1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Marvell CN10K DRAM Subsystem (DSS) Performance Monitor Driver
4 *
5 * Copyright (C) 2021-2024 Marvell.
6 */
7
8 #include <linux/init.h>
9 #include <linux/io.h>
10 #include <linux/module.h>
11 #include <linux/of.h>
12 #include <linux/perf_event.h>
13 #include <linux/hrtimer.h>
14 #include <linux/acpi.h>
15 #include <linux/platform_device.h>
16
17 /* Performance Counters Operating Mode Control Registers */
18 #define CN10K_DDRC_PERF_CNT_OP_MODE_CTRL 0x8020
19 #define ODY_DDRC_PERF_CNT_OP_MODE_CTRL 0x20020
20 #define OP_MODE_CTRL_VAL_MANUAL 0x1
21
22 /* Performance Counters Start Operation Control Registers */
23 #define CN10K_DDRC_PERF_CNT_START_OP_CTRL 0x8028
24 #define ODY_DDRC_PERF_CNT_START_OP_CTRL 0x200A0
25 #define START_OP_CTRL_VAL_START 0x1ULL
26 #define START_OP_CTRL_VAL_ACTIVE 0x2
27
28 /* Performance Counters End Operation Control Registers */
29 #define CN10K_DDRC_PERF_CNT_END_OP_CTRL 0x8030
30 #define ODY_DDRC_PERF_CNT_END_OP_CTRL 0x200E0
31 #define END_OP_CTRL_VAL_END 0x1ULL
32
33 /* Performance Counters End Status Registers */
34 #define CN10K_DDRC_PERF_CNT_END_STATUS 0x8038
35 #define ODY_DDRC_PERF_CNT_END_STATUS 0x20120
36 #define END_STATUS_VAL_END_TIMER_MODE_END 0x1
37
38 /* Performance Counters Configuration Registers */
39 #define CN10K_DDRC_PERF_CFG_BASE 0x8040
40 #define ODY_DDRC_PERF_CFG_BASE 0x20160
41
42 /* 8 Generic event counter + 2 fixed event counters */
43 #define DDRC_PERF_NUM_GEN_COUNTERS 8
44 #define DDRC_PERF_NUM_FIX_COUNTERS 2
45 #define DDRC_PERF_READ_COUNTER_IDX DDRC_PERF_NUM_GEN_COUNTERS
46 #define DDRC_PERF_WRITE_COUNTER_IDX (DDRC_PERF_NUM_GEN_COUNTERS + 1)
47 #define DDRC_PERF_NUM_COUNTERS (DDRC_PERF_NUM_GEN_COUNTERS + \
48 DDRC_PERF_NUM_FIX_COUNTERS)
49
50 /* Generic event counter registers */
51 #define DDRC_PERF_CFG(base, n) ((base) + 8 * (n))
52 #define EVENT_ENABLE BIT_ULL(63)
53
54 /* Two dedicated event counters for DDR reads and writes */
55 #define EVENT_DDR_READS 101
56 #define EVENT_DDR_WRITES 100
57
58 #define DDRC_PERF_REG(base, n) ((base) + 8 * (n))
59 /*
60 * programmable events IDs in programmable event counters.
61 * DO NOT change these event-id numbers, they are used to
62 * program event bitmap in h/w.
63 */
64 #define EVENT_DFI_CMD_IS_RETRY 61
65 #define EVENT_RD_UC_ECC_ERROR 60
66 #define EVENT_RD_CRC_ERROR 59
67 #define EVENT_CAPAR_ERROR 58
68 #define EVENT_WR_CRC_ERROR 57
69 #define EVENT_DFI_PARITY_POISON 56
70 #define EVENT_RETRY_FIFO_FULL 46
71 #define EVENT_DFI_CYCLES 45
72
73 #define EVENT_OP_IS_ZQLATCH 55
74 #define EVENT_OP_IS_ZQSTART 54
75 #define EVENT_OP_IS_TCR_MRR 53
76 #define EVENT_OP_IS_DQSOSC_MRR 52
77 #define EVENT_OP_IS_DQSOSC_MPC 51
78 #define EVENT_VISIBLE_WIN_LIMIT_REACHED_WR 50
79 #define EVENT_VISIBLE_WIN_LIMIT_REACHED_RD 49
80 #define EVENT_BSM_STARVATION 48
81 #define EVENT_BSM_ALLOC 47
82 #define EVENT_LPR_REQ_WITH_NOCREDIT 46
83 #define EVENT_HPR_REQ_WITH_NOCREDIT 45
84 #define EVENT_OP_IS_ZQCS 44
85 #define EVENT_OP_IS_ZQCL 43
86 #define EVENT_OP_IS_LOAD_MODE 42
87 #define EVENT_OP_IS_SPEC_REF 41
88 #define EVENT_OP_IS_CRIT_REF 40
89 #define EVENT_OP_IS_REFRESH 39
90 #define EVENT_OP_IS_ENTER_MPSM 35
91 #define EVENT_OP_IS_ENTER_POWERDOWN 31
92 #define EVENT_OP_IS_ENTER_SELFREF 27
93 #define EVENT_WAW_HAZARD 26
94 #define EVENT_RAW_HAZARD 25
95 #define EVENT_WAR_HAZARD 24
96 #define EVENT_WRITE_COMBINE 23
97 #define EVENT_RDWR_TRANSITIONS 22
98 #define EVENT_PRECHARGE_FOR_OTHER 21
99 #define EVENT_PRECHARGE_FOR_RDWR 20
100 #define EVENT_OP_IS_PRECHARGE 19
101 #define EVENT_OP_IS_MWR 18
102 #define EVENT_OP_IS_WR 17
103 #define EVENT_OP_IS_RD 16
104 #define EVENT_OP_IS_RD_ACTIVATE 15
105 #define EVENT_OP_IS_RD_OR_WR 14
106 #define EVENT_OP_IS_ACTIVATE 13
107 #define EVENT_WR_XACT_WHEN_CRITICAL 12
108 #define EVENT_LPR_XACT_WHEN_CRITICAL 11
109 #define EVENT_HPR_XACT_WHEN_CRITICAL 10
110 #define EVENT_DFI_RD_DATA_CYCLES 9
111 #define EVENT_DFI_WR_DATA_CYCLES 8
112 #define EVENT_ACT_BYPASS 7
113 #define EVENT_READ_BYPASS 6
114 #define EVENT_HIF_HI_PRI_RD 5
115 #define EVENT_HIF_RMW 4
116 #define EVENT_HIF_RD 3
117 #define EVENT_HIF_WR 2
118 #define EVENT_HIF_RD_OR_WR 1
119
120 /* Event counter value registers */
121 #define CN10K_DDRC_PERF_CNT_VALUE_BASE 0x8080
122 #define ODY_DDRC_PERF_CNT_VALUE_BASE 0x201C0
123
124 /* Fixed event counter enable/disable register */
125 #define CN10K_DDRC_PERF_CNT_FREERUN_EN 0x80C0
126 #define DDRC_PERF_FREERUN_WRITE_EN 0x1
127 #define DDRC_PERF_FREERUN_READ_EN 0x2
128
129 /* Fixed event counter control register */
130 #define CN10K_DDRC_PERF_CNT_FREERUN_CTRL 0x80C8
131 #define ODY_DDRC_PERF_CNT_FREERUN_CTRL 0x20240
132 #define DDRC_FREERUN_WRITE_CNT_CLR 0x1
133 #define DDRC_FREERUN_READ_CNT_CLR 0x2
134
135 /* Fixed event counter clear register, defined only for Odyssey */
136 #define ODY_DDRC_PERF_CNT_FREERUN_CLR 0x20248
137
138 #define DDRC_PERF_CNT_VALUE_OVERFLOW BIT_ULL(48)
139 #define DDRC_PERF_CNT_MAX_VALUE GENMASK_ULL(48, 0)
140
141 /* Fixed event counter value register */
142 #define CN10K_DDRC_PERF_CNT_VALUE_WR_OP 0x80D0
143 #define CN10K_DDRC_PERF_CNT_VALUE_RD_OP 0x80D8
144 #define ODY_DDRC_PERF_CNT_VALUE_WR_OP 0x20250
145 #define ODY_DDRC_PERF_CNT_VALUE_RD_OP 0x20258
146
147 struct cn10k_ddr_pmu {
148 struct pmu pmu;
149 void __iomem *base;
150 const struct ddr_pmu_platform_data *p_data;
151 const struct ddr_pmu_ops *ops;
152 unsigned int cpu;
153 struct device *dev;
154 int active_events;
155 struct perf_event *events[DDRC_PERF_NUM_COUNTERS];
156 struct hrtimer hrtimer;
157 struct hlist_node node;
158 };
159
160 struct ddr_pmu_ops {
161 void (*enable_read_freerun_counter)(struct cn10k_ddr_pmu *pmu,
162 bool enable);
163 void (*enable_write_freerun_counter)(struct cn10k_ddr_pmu *pmu,
164 bool enable);
165 void (*clear_read_freerun_counter)(struct cn10k_ddr_pmu *pmu);
166 void (*clear_write_freerun_counter)(struct cn10k_ddr_pmu *pmu);
167 void (*pmu_overflow_handler)(struct cn10k_ddr_pmu *pmu, int evt_idx);
168 };
169
170 #define to_cn10k_ddr_pmu(p) container_of(p, struct cn10k_ddr_pmu, pmu)
171
172 struct ddr_pmu_platform_data {
173 u64 counter_overflow_val;
174 u64 counter_max_val;
175 u64 cnt_base;
176 u64 cfg_base;
177 u64 cnt_op_mode_ctrl;
178 u64 cnt_start_op_ctrl;
179 u64 cnt_end_op_ctrl;
180 u64 cnt_end_status;
181 u64 cnt_freerun_en;
182 u64 cnt_freerun_ctrl;
183 u64 cnt_freerun_clr;
184 u64 cnt_value_wr_op;
185 u64 cnt_value_rd_op;
186 bool is_cn10k;
187 bool is_ody;
188 };
189
cn10k_ddr_pmu_event_show(struct device * dev,struct device_attribute * attr,char * page)190 static ssize_t cn10k_ddr_pmu_event_show(struct device *dev,
191 struct device_attribute *attr,
192 char *page)
193 {
194 struct perf_pmu_events_attr *pmu_attr;
195
196 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
197 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
198
199 }
200
201 #define CN10K_DDR_PMU_EVENT_ATTR(_name, _id) \
202 PMU_EVENT_ATTR_ID(_name, cn10k_ddr_pmu_event_show, _id)
203
204 static struct attribute *cn10k_ddr_perf_events_attrs[] = {
205 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR),
206 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR),
207 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD),
208 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW),
209 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD),
210 CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS),
211 CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS),
212 CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_wr_data_access, EVENT_DFI_WR_DATA_CYCLES),
213 CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_rd_data_access, EVENT_DFI_RD_DATA_CYCLES),
214 CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access,
215 EVENT_HPR_XACT_WHEN_CRITICAL),
216 CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access,
217 EVENT_LPR_XACT_WHEN_CRITICAL),
218 CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access,
219 EVENT_WR_XACT_WHEN_CRITICAL),
220 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE),
221 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access, EVENT_OP_IS_RD_OR_WR),
222 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access, EVENT_OP_IS_RD_ACTIVATE),
223 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD),
224 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR),
225 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR),
226 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE),
227 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr, EVENT_PRECHARGE_FOR_RDWR),
228 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other,
229 EVENT_PRECHARGE_FOR_OTHER),
230 CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS),
231 CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE),
232 CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD),
233 CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD),
234 CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD),
235 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF),
236 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown, EVENT_OP_IS_ENTER_POWERDOWN),
237 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_mpsm, EVENT_OP_IS_ENTER_MPSM),
238 CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH),
239 CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF),
240 CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF),
241 CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE),
242 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqcl, EVENT_OP_IS_ZQCL),
243 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_wr_access, EVENT_OP_IS_ZQCS),
244 CN10K_DDR_PMU_EVENT_ATTR(ddr_hpr_req_with_nocredit,
245 EVENT_HPR_REQ_WITH_NOCREDIT),
246 CN10K_DDR_PMU_EVENT_ATTR(ddr_lpr_req_with_nocredit,
247 EVENT_LPR_REQ_WITH_NOCREDIT),
248 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_alloc, EVENT_BSM_ALLOC),
249 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_starvation, EVENT_BSM_STARVATION),
250 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd,
251 EVENT_VISIBLE_WIN_LIMIT_REACHED_RD),
252 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr,
253 EVENT_VISIBLE_WIN_LIMIT_REACHED_WR),
254 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_OP_IS_DQSOSC_MPC),
255 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_OP_IS_DQSOSC_MRR),
256 CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_OP_IS_TCR_MRR),
257 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_OP_IS_ZQSTART),
258 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_OP_IS_ZQLATCH),
259 /* Free run event counters */
260 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS),
261 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES),
262 NULL
263 };
264
265 static struct attribute *odyssey_ddr_perf_events_attrs[] = {
266 /* Programmable */
267 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR),
268 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR),
269 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD),
270 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW),
271 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD),
272 CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS),
273 CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS),
274 CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_wr_data_access,
275 EVENT_DFI_WR_DATA_CYCLES),
276 CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_rd_data_access,
277 EVENT_DFI_RD_DATA_CYCLES),
278 CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access,
279 EVENT_HPR_XACT_WHEN_CRITICAL),
280 CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access,
281 EVENT_LPR_XACT_WHEN_CRITICAL),
282 CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access,
283 EVENT_WR_XACT_WHEN_CRITICAL),
284 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE),
285 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access,
286 EVENT_OP_IS_RD_OR_WR),
287 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access,
288 EVENT_OP_IS_RD_ACTIVATE),
289 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD),
290 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR),
291 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR),
292 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE),
293 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr,
294 EVENT_PRECHARGE_FOR_RDWR),
295 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other,
296 EVENT_PRECHARGE_FOR_OTHER),
297 CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS),
298 CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE),
299 CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD),
300 CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD),
301 CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD),
302 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF),
303 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown,
304 EVENT_OP_IS_ENTER_POWERDOWN),
305 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_mpsm, EVENT_OP_IS_ENTER_MPSM),
306 CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH),
307 CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF),
308 CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF),
309 CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE),
310 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqcl, EVENT_OP_IS_ZQCL),
311 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_wr_access, EVENT_OP_IS_ZQCS),
312 CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_cycles, EVENT_DFI_CYCLES),
313 CN10K_DDR_PMU_EVENT_ATTR(ddr_retry_fifo_full,
314 EVENT_RETRY_FIFO_FULL),
315 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_alloc, EVENT_BSM_ALLOC),
316 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_starvation, EVENT_BSM_STARVATION),
317 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd,
318 EVENT_VISIBLE_WIN_LIMIT_REACHED_RD),
319 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr,
320 EVENT_VISIBLE_WIN_LIMIT_REACHED_WR),
321 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_OP_IS_DQSOSC_MPC),
322 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_OP_IS_DQSOSC_MRR),
323 CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_OP_IS_TCR_MRR),
324 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_OP_IS_ZQSTART),
325 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_OP_IS_ZQLATCH),
326 CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_parity_poison,
327 EVENT_DFI_PARITY_POISON),
328 CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_crc_error, EVENT_WR_CRC_ERROR),
329 CN10K_DDR_PMU_EVENT_ATTR(ddr_capar_error, EVENT_CAPAR_ERROR),
330 CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_crc_error, EVENT_RD_CRC_ERROR),
331 CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_uc_ecc_error, EVENT_RD_UC_ECC_ERROR),
332 CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_cmd_is_retry, EVENT_DFI_CMD_IS_RETRY),
333 /* Free run event counters */
334 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS),
335 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES),
336 NULL
337 };
338
339 static struct attribute_group odyssey_ddr_perf_events_attr_group = {
340 .name = "events",
341 .attrs = odyssey_ddr_perf_events_attrs,
342 };
343
344 static struct attribute_group cn10k_ddr_perf_events_attr_group = {
345 .name = "events",
346 .attrs = cn10k_ddr_perf_events_attrs,
347 };
348
349 PMU_FORMAT_ATTR(event, "config:0-8");
350
351 static struct attribute *cn10k_ddr_perf_format_attrs[] = {
352 &format_attr_event.attr,
353 NULL,
354 };
355
356 static struct attribute_group cn10k_ddr_perf_format_attr_group = {
357 .name = "format",
358 .attrs = cn10k_ddr_perf_format_attrs,
359 };
360
cn10k_ddr_perf_cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)361 static ssize_t cn10k_ddr_perf_cpumask_show(struct device *dev,
362 struct device_attribute *attr,
363 char *buf)
364 {
365 struct cn10k_ddr_pmu *pmu = dev_get_drvdata(dev);
366
367 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
368 }
369
370 static struct device_attribute cn10k_ddr_perf_cpumask_attr =
371 __ATTR(cpumask, 0444, cn10k_ddr_perf_cpumask_show, NULL);
372
373 static struct attribute *cn10k_ddr_perf_cpumask_attrs[] = {
374 &cn10k_ddr_perf_cpumask_attr.attr,
375 NULL,
376 };
377
378 static struct attribute_group cn10k_ddr_perf_cpumask_attr_group = {
379 .attrs = cn10k_ddr_perf_cpumask_attrs,
380 };
381
382 static const struct attribute_group *cn10k_attr_groups[] = {
383 &cn10k_ddr_perf_events_attr_group,
384 &cn10k_ddr_perf_format_attr_group,
385 &cn10k_ddr_perf_cpumask_attr_group,
386 NULL,
387 };
388
389 static const struct attribute_group *odyssey_attr_groups[] = {
390 &odyssey_ddr_perf_events_attr_group,
391 &cn10k_ddr_perf_format_attr_group,
392 &cn10k_ddr_perf_cpumask_attr_group,
393 NULL
394 };
395
396 /* Default poll timeout is 100 sec, which is very sufficient for
397 * 48 bit counter incremented max at 5.6 GT/s, which may take many
398 * hours to overflow.
399 */
400 static unsigned long cn10k_ddr_pmu_poll_period_sec = 100;
401 module_param_named(poll_period_sec, cn10k_ddr_pmu_poll_period_sec, ulong, 0644);
402
cn10k_ddr_pmu_timer_period(void)403 static ktime_t cn10k_ddr_pmu_timer_period(void)
404 {
405 return ms_to_ktime((u64)cn10k_ddr_pmu_poll_period_sec * USEC_PER_SEC);
406 }
407
ddr_perf_get_event_bitmap(int eventid,u64 * event_bitmap,struct cn10k_ddr_pmu * ddr_pmu)408 static int ddr_perf_get_event_bitmap(int eventid, u64 *event_bitmap,
409 struct cn10k_ddr_pmu *ddr_pmu)
410 {
411 int err = 0;
412
413 switch (eventid) {
414 case EVENT_DFI_PARITY_POISON ...EVENT_DFI_CMD_IS_RETRY:
415 if (!ddr_pmu->p_data->is_ody) {
416 err = -EINVAL;
417 break;
418 }
419 fallthrough;
420 case EVENT_HIF_RD_OR_WR ... EVENT_WAW_HAZARD:
421 case EVENT_OP_IS_REFRESH ... EVENT_OP_IS_ZQLATCH:
422 *event_bitmap = (1ULL << (eventid - 1));
423 break;
424 case EVENT_OP_IS_ENTER_SELFREF:
425 case EVENT_OP_IS_ENTER_POWERDOWN:
426 case EVENT_OP_IS_ENTER_MPSM:
427 *event_bitmap = (0xFULL << (eventid - 1));
428 break;
429 default:
430 err = -EINVAL;
431 }
432
433 if (err)
434 pr_err("%s Invalid eventid %d\n", __func__, eventid);
435 return err;
436 }
437
cn10k_ddr_perf_alloc_counter(struct cn10k_ddr_pmu * pmu,struct perf_event * event)438 static int cn10k_ddr_perf_alloc_counter(struct cn10k_ddr_pmu *pmu,
439 struct perf_event *event)
440 {
441 u8 config = event->attr.config;
442 int i;
443
444 /* DDR read free-run counter index */
445 if (config == EVENT_DDR_READS) {
446 pmu->events[DDRC_PERF_READ_COUNTER_IDX] = event;
447 return DDRC_PERF_READ_COUNTER_IDX;
448 }
449
450 /* DDR write free-run counter index */
451 if (config == EVENT_DDR_WRITES) {
452 pmu->events[DDRC_PERF_WRITE_COUNTER_IDX] = event;
453 return DDRC_PERF_WRITE_COUNTER_IDX;
454 }
455
456 /* Allocate DDR generic counters */
457 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
458 if (pmu->events[i] == NULL) {
459 pmu->events[i] = event;
460 return i;
461 }
462 }
463
464 return -ENOENT;
465 }
466
cn10k_ddr_perf_free_counter(struct cn10k_ddr_pmu * pmu,int counter)467 static void cn10k_ddr_perf_free_counter(struct cn10k_ddr_pmu *pmu, int counter)
468 {
469 pmu->events[counter] = NULL;
470 }
471
cn10k_ddr_perf_event_init(struct perf_event * event)472 static int cn10k_ddr_perf_event_init(struct perf_event *event)
473 {
474 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
475 struct hw_perf_event *hwc = &event->hw;
476
477 if (event->attr.type != event->pmu->type)
478 return -ENOENT;
479
480 if (is_sampling_event(event)) {
481 dev_info(pmu->dev, "Sampling not supported!\n");
482 return -EOPNOTSUPP;
483 }
484
485 if (event->cpu < 0) {
486 dev_warn(pmu->dev, "Can't provide per-task data!\n");
487 return -EOPNOTSUPP;
488 }
489
490 /* We must NOT create groups containing mixed PMUs */
491 if (event->group_leader->pmu != event->pmu &&
492 !is_software_event(event->group_leader))
493 return -EINVAL;
494
495 /* Set ownership of event to one CPU, same event can not be observed
496 * on multiple cpus at same time.
497 */
498 event->cpu = pmu->cpu;
499 hwc->idx = -1;
500 return 0;
501 }
502
cn10k_ddr_perf_counter_start(struct cn10k_ddr_pmu * ddr_pmu,int counter)503 static void cn10k_ddr_perf_counter_start(struct cn10k_ddr_pmu *ddr_pmu,
504 int counter)
505 {
506 const struct ddr_pmu_platform_data *p_data = ddr_pmu->p_data;
507 u64 ctrl_reg = p_data->cnt_start_op_ctrl;
508
509 writeq_relaxed(START_OP_CTRL_VAL_START, ddr_pmu->base +
510 DDRC_PERF_REG(ctrl_reg, counter));
511 }
512
cn10k_ddr_perf_counter_stop(struct cn10k_ddr_pmu * ddr_pmu,int counter)513 static void cn10k_ddr_perf_counter_stop(struct cn10k_ddr_pmu *ddr_pmu,
514 int counter)
515 {
516 const struct ddr_pmu_platform_data *p_data = ddr_pmu->p_data;
517 u64 ctrl_reg = p_data->cnt_end_op_ctrl;
518
519 writeq_relaxed(END_OP_CTRL_VAL_END, ddr_pmu->base +
520 DDRC_PERF_REG(ctrl_reg, counter));
521 }
522
cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu * pmu,int counter,bool enable)523 static void cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu *pmu,
524 int counter, bool enable)
525 {
526 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
527 u64 ctrl_reg = pmu->p_data->cnt_op_mode_ctrl;
528 const struct ddr_pmu_ops *ops = pmu->ops;
529 bool is_ody = pmu->p_data->is_ody;
530 u32 reg;
531 u64 val;
532
533 if (counter > DDRC_PERF_NUM_COUNTERS) {
534 pr_err("Error: unsupported counter %d\n", counter);
535 return;
536 }
537
538 if (counter < DDRC_PERF_NUM_GEN_COUNTERS) {
539 reg = DDRC_PERF_CFG(p_data->cfg_base, counter);
540 val = readq_relaxed(pmu->base + reg);
541
542 if (enable)
543 val |= EVENT_ENABLE;
544 else
545 val &= ~EVENT_ENABLE;
546
547 writeq_relaxed(val, pmu->base + reg);
548
549 if (is_ody) {
550 if (enable) {
551 /*
552 * Setup the PMU counter to work in
553 * manual mode
554 */
555 reg = DDRC_PERF_REG(ctrl_reg, counter);
556 writeq_relaxed(OP_MODE_CTRL_VAL_MANUAL,
557 pmu->base + reg);
558
559 cn10k_ddr_perf_counter_start(pmu, counter);
560 } else {
561 cn10k_ddr_perf_counter_stop(pmu, counter);
562 }
563 }
564 } else {
565 if (counter == DDRC_PERF_READ_COUNTER_IDX)
566 ops->enable_read_freerun_counter(pmu, enable);
567 else
568 ops->enable_write_freerun_counter(pmu, enable);
569 }
570 }
571
cn10k_ddr_perf_read_counter(struct cn10k_ddr_pmu * pmu,int counter)572 static u64 cn10k_ddr_perf_read_counter(struct cn10k_ddr_pmu *pmu, int counter)
573 {
574 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
575 u64 val;
576
577 if (counter == DDRC_PERF_READ_COUNTER_IDX)
578 return readq_relaxed(pmu->base +
579 p_data->cnt_value_rd_op);
580
581 if (counter == DDRC_PERF_WRITE_COUNTER_IDX)
582 return readq_relaxed(pmu->base +
583 p_data->cnt_value_wr_op);
584
585 val = readq_relaxed(pmu->base +
586 DDRC_PERF_REG(p_data->cnt_base, counter));
587 return val;
588 }
589
cn10k_ddr_perf_event_update(struct perf_event * event)590 static void cn10k_ddr_perf_event_update(struct perf_event *event)
591 {
592 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
593 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
594 struct hw_perf_event *hwc = &event->hw;
595 u64 prev_count, new_count, mask;
596
597 do {
598 prev_count = local64_read(&hwc->prev_count);
599 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
600 } while (local64_xchg(&hwc->prev_count, new_count) != prev_count);
601
602 mask = p_data->counter_max_val;
603
604 local64_add((new_count - prev_count) & mask, &event->count);
605 }
606
cn10k_ddr_perf_event_start(struct perf_event * event,int flags)607 static void cn10k_ddr_perf_event_start(struct perf_event *event, int flags)
608 {
609 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
610 struct hw_perf_event *hwc = &event->hw;
611 int counter = hwc->idx;
612
613 local64_set(&hwc->prev_count, 0);
614
615 cn10k_ddr_perf_counter_enable(pmu, counter, true);
616
617 hwc->state = 0;
618 }
619
cn10k_ddr_perf_event_add(struct perf_event * event,int flags)620 static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags)
621 {
622 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
623 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
624 const struct ddr_pmu_ops *ops = pmu->ops;
625 struct hw_perf_event *hwc = &event->hw;
626 u8 config = event->attr.config;
627 int counter, ret;
628 u32 reg_offset;
629 u64 val;
630
631 counter = cn10k_ddr_perf_alloc_counter(pmu, event);
632 if (counter < 0)
633 return -EAGAIN;
634
635 pmu->active_events++;
636 hwc->idx = counter;
637
638 if (pmu->active_events == 1)
639 hrtimer_start(&pmu->hrtimer, cn10k_ddr_pmu_timer_period(),
640 HRTIMER_MODE_REL_PINNED);
641
642 if (counter < DDRC_PERF_NUM_GEN_COUNTERS) {
643 /* Generic counters, configure event id */
644 reg_offset = DDRC_PERF_CFG(p_data->cfg_base, counter);
645 ret = ddr_perf_get_event_bitmap(config, &val, pmu);
646 if (ret)
647 return ret;
648
649 writeq_relaxed(val, pmu->base + reg_offset);
650 } else {
651 /* fixed event counter, clear counter value */
652 if (counter == DDRC_PERF_READ_COUNTER_IDX)
653 ops->clear_read_freerun_counter(pmu);
654 else
655 ops->clear_write_freerun_counter(pmu);
656 }
657
658 hwc->state |= PERF_HES_STOPPED;
659
660 if (flags & PERF_EF_START)
661 cn10k_ddr_perf_event_start(event, flags);
662
663 return 0;
664 }
665
cn10k_ddr_perf_event_stop(struct perf_event * event,int flags)666 static void cn10k_ddr_perf_event_stop(struct perf_event *event, int flags)
667 {
668 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
669 struct hw_perf_event *hwc = &event->hw;
670 int counter = hwc->idx;
671
672 cn10k_ddr_perf_counter_enable(pmu, counter, false);
673
674 if (flags & PERF_EF_UPDATE)
675 cn10k_ddr_perf_event_update(event);
676
677 hwc->state |= PERF_HES_STOPPED;
678 }
679
cn10k_ddr_perf_event_del(struct perf_event * event,int flags)680 static void cn10k_ddr_perf_event_del(struct perf_event *event, int flags)
681 {
682 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
683 struct hw_perf_event *hwc = &event->hw;
684 int counter = hwc->idx;
685
686 cn10k_ddr_perf_event_stop(event, PERF_EF_UPDATE);
687
688 cn10k_ddr_perf_free_counter(pmu, counter);
689 pmu->active_events--;
690 hwc->idx = -1;
691
692 /* Cancel timer when no events to capture */
693 if (pmu->active_events == 0)
694 hrtimer_cancel(&pmu->hrtimer);
695 }
696
cn10k_ddr_perf_pmu_enable(struct pmu * pmu)697 static void cn10k_ddr_perf_pmu_enable(struct pmu *pmu)
698 {
699 struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu);
700 const struct ddr_pmu_platform_data *p_data = ddr_pmu->p_data;
701
702 writeq_relaxed(START_OP_CTRL_VAL_START, ddr_pmu->base +
703 p_data->cnt_start_op_ctrl);
704 }
705
cn10k_ddr_perf_pmu_disable(struct pmu * pmu)706 static void cn10k_ddr_perf_pmu_disable(struct pmu *pmu)
707 {
708 struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu);
709 const struct ddr_pmu_platform_data *p_data = ddr_pmu->p_data;
710
711 writeq_relaxed(END_OP_CTRL_VAL_END, ddr_pmu->base +
712 p_data->cnt_end_op_ctrl);
713 }
714
cn10k_ddr_perf_event_update_all(struct cn10k_ddr_pmu * pmu)715 static void cn10k_ddr_perf_event_update_all(struct cn10k_ddr_pmu *pmu)
716 {
717 struct hw_perf_event *hwc;
718 int i;
719
720 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
721 if (pmu->events[i] == NULL)
722 continue;
723
724 cn10k_ddr_perf_event_update(pmu->events[i]);
725 }
726
727 /* Reset previous count as h/w counter are reset */
728 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
729 if (pmu->events[i] == NULL)
730 continue;
731
732 hwc = &pmu->events[i]->hw;
733 local64_set(&hwc->prev_count, 0);
734 }
735 }
736
ddr_pmu_enable_read_freerun(struct cn10k_ddr_pmu * pmu,bool enable)737 static void ddr_pmu_enable_read_freerun(struct cn10k_ddr_pmu *pmu, bool enable)
738 {
739 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
740 u64 val;
741
742 val = readq_relaxed(pmu->base + p_data->cnt_freerun_en);
743 if (enable)
744 val |= DDRC_PERF_FREERUN_READ_EN;
745 else
746 val &= ~DDRC_PERF_FREERUN_READ_EN;
747
748 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_en);
749 }
750
ddr_pmu_enable_write_freerun(struct cn10k_ddr_pmu * pmu,bool enable)751 static void ddr_pmu_enable_write_freerun(struct cn10k_ddr_pmu *pmu, bool enable)
752 {
753 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
754 u64 val;
755
756 val = readq_relaxed(pmu->base + p_data->cnt_freerun_en);
757 if (enable)
758 val |= DDRC_PERF_FREERUN_WRITE_EN;
759 else
760 val &= ~DDRC_PERF_FREERUN_WRITE_EN;
761
762 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_en);
763 }
764
ddr_pmu_read_clear_freerun(struct cn10k_ddr_pmu * pmu)765 static void ddr_pmu_read_clear_freerun(struct cn10k_ddr_pmu *pmu)
766 {
767 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
768 u64 val;
769
770 val = DDRC_FREERUN_READ_CNT_CLR;
771 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_ctrl);
772 }
773
ddr_pmu_write_clear_freerun(struct cn10k_ddr_pmu * pmu)774 static void ddr_pmu_write_clear_freerun(struct cn10k_ddr_pmu *pmu)
775 {
776 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
777 u64 val;
778
779 val = DDRC_FREERUN_WRITE_CNT_CLR;
780 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_ctrl);
781 }
782
ddr_pmu_overflow_hander(struct cn10k_ddr_pmu * pmu,int evt_idx)783 static void ddr_pmu_overflow_hander(struct cn10k_ddr_pmu *pmu, int evt_idx)
784 {
785 cn10k_ddr_perf_event_update_all(pmu);
786 cn10k_ddr_perf_pmu_disable(&pmu->pmu);
787 cn10k_ddr_perf_pmu_enable(&pmu->pmu);
788 }
789
ddr_pmu_ody_enable_read_freerun(struct cn10k_ddr_pmu * pmu,bool enable)790 static void ddr_pmu_ody_enable_read_freerun(struct cn10k_ddr_pmu *pmu,
791 bool enable)
792 {
793 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
794 u64 val;
795
796 val = readq_relaxed(pmu->base + p_data->cnt_freerun_ctrl);
797 if (enable)
798 val |= DDRC_PERF_FREERUN_READ_EN;
799 else
800 val &= ~DDRC_PERF_FREERUN_READ_EN;
801
802 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_ctrl);
803 }
804
ddr_pmu_ody_enable_write_freerun(struct cn10k_ddr_pmu * pmu,bool enable)805 static void ddr_pmu_ody_enable_write_freerun(struct cn10k_ddr_pmu *pmu,
806 bool enable)
807 {
808 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
809 u64 val;
810
811 val = readq_relaxed(pmu->base + p_data->cnt_freerun_ctrl);
812 if (enable)
813 val |= DDRC_PERF_FREERUN_WRITE_EN;
814 else
815 val &= ~DDRC_PERF_FREERUN_WRITE_EN;
816
817 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_ctrl);
818 }
819
ddr_pmu_ody_read_clear_freerun(struct cn10k_ddr_pmu * pmu)820 static void ddr_pmu_ody_read_clear_freerun(struct cn10k_ddr_pmu *pmu)
821 {
822 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
823 u64 val;
824
825 val = DDRC_FREERUN_READ_CNT_CLR;
826 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_clr);
827 }
828
ddr_pmu_ody_write_clear_freerun(struct cn10k_ddr_pmu * pmu)829 static void ddr_pmu_ody_write_clear_freerun(struct cn10k_ddr_pmu *pmu)
830 {
831 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
832 u64 val;
833
834 val = DDRC_FREERUN_WRITE_CNT_CLR;
835 writeq_relaxed(val, pmu->base + p_data->cnt_freerun_clr);
836 }
837
ddr_pmu_ody_overflow_hander(struct cn10k_ddr_pmu * pmu,int evt_idx)838 static void ddr_pmu_ody_overflow_hander(struct cn10k_ddr_pmu *pmu, int evt_idx)
839 {
840 /*
841 * On reaching the maximum value of the counter, the counter freezes
842 * there. The particular event is updated and the respective counter
843 * is stopped and started again so that it starts counting from zero
844 */
845 cn10k_ddr_perf_event_update(pmu->events[evt_idx]);
846 cn10k_ddr_perf_counter_stop(pmu, evt_idx);
847 cn10k_ddr_perf_counter_start(pmu, evt_idx);
848 }
849
cn10k_ddr_pmu_overflow_handler(struct cn10k_ddr_pmu * pmu)850 static irqreturn_t cn10k_ddr_pmu_overflow_handler(struct cn10k_ddr_pmu *pmu)
851 {
852 const struct ddr_pmu_platform_data *p_data = pmu->p_data;
853 const struct ddr_pmu_ops *ops = pmu->ops;
854 struct perf_event *event;
855 struct hw_perf_event *hwc;
856 u64 prev_count, new_count;
857 u64 value;
858 int i;
859
860 event = pmu->events[DDRC_PERF_READ_COUNTER_IDX];
861 if (event) {
862 hwc = &event->hw;
863 prev_count = local64_read(&hwc->prev_count);
864 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
865
866 /* Overflow condition is when new count less than
867 * previous count
868 */
869 if (new_count < prev_count)
870 cn10k_ddr_perf_event_update(event);
871 }
872
873 event = pmu->events[DDRC_PERF_WRITE_COUNTER_IDX];
874 if (event) {
875 hwc = &event->hw;
876 prev_count = local64_read(&hwc->prev_count);
877 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
878
879 /* Overflow condition is when new count less than
880 * previous count
881 */
882 if (new_count < prev_count)
883 cn10k_ddr_perf_event_update(event);
884 }
885
886 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
887 if (pmu->events[i] == NULL)
888 continue;
889
890 value = cn10k_ddr_perf_read_counter(pmu, i);
891 if (value == p_data->counter_max_val) {
892 pr_info("Counter-(%d) reached max value\n", i);
893 ops->pmu_overflow_handler(pmu, i);
894 }
895 }
896
897 return IRQ_HANDLED;
898 }
899
cn10k_ddr_pmu_timer_handler(struct hrtimer * hrtimer)900 static enum hrtimer_restart cn10k_ddr_pmu_timer_handler(struct hrtimer *hrtimer)
901 {
902 struct cn10k_ddr_pmu *pmu = container_of(hrtimer, struct cn10k_ddr_pmu,
903 hrtimer);
904 unsigned long flags;
905
906 local_irq_save(flags);
907 cn10k_ddr_pmu_overflow_handler(pmu);
908 local_irq_restore(flags);
909
910 hrtimer_forward_now(hrtimer, cn10k_ddr_pmu_timer_period());
911 return HRTIMER_RESTART;
912 }
913
cn10k_ddr_pmu_offline_cpu(unsigned int cpu,struct hlist_node * node)914 static int cn10k_ddr_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
915 {
916 struct cn10k_ddr_pmu *pmu = hlist_entry_safe(node, struct cn10k_ddr_pmu,
917 node);
918 unsigned int target;
919
920 if (cpu != pmu->cpu)
921 return 0;
922
923 target = cpumask_any_but(cpu_online_mask, cpu);
924 if (target >= nr_cpu_ids)
925 return 0;
926
927 perf_pmu_migrate_context(&pmu->pmu, cpu, target);
928 pmu->cpu = target;
929 return 0;
930 }
931
932 static const struct ddr_pmu_ops ddr_pmu_ops = {
933 .enable_read_freerun_counter = ddr_pmu_enable_read_freerun,
934 .enable_write_freerun_counter = ddr_pmu_enable_write_freerun,
935 .clear_read_freerun_counter = ddr_pmu_read_clear_freerun,
936 .clear_write_freerun_counter = ddr_pmu_write_clear_freerun,
937 .pmu_overflow_handler = ddr_pmu_overflow_hander,
938 };
939
940 #if defined(CONFIG_ACPI) || defined(CONFIG_OF)
941 static const struct ddr_pmu_platform_data cn10k_ddr_pmu_pdata = {
942 .counter_overflow_val = BIT_ULL(48),
943 .counter_max_val = GENMASK_ULL(48, 0),
944 .cnt_base = CN10K_DDRC_PERF_CNT_VALUE_BASE,
945 .cfg_base = CN10K_DDRC_PERF_CFG_BASE,
946 .cnt_op_mode_ctrl = CN10K_DDRC_PERF_CNT_OP_MODE_CTRL,
947 .cnt_start_op_ctrl = CN10K_DDRC_PERF_CNT_START_OP_CTRL,
948 .cnt_end_op_ctrl = CN10K_DDRC_PERF_CNT_END_OP_CTRL,
949 .cnt_end_status = CN10K_DDRC_PERF_CNT_END_STATUS,
950 .cnt_freerun_en = CN10K_DDRC_PERF_CNT_FREERUN_EN,
951 .cnt_freerun_ctrl = CN10K_DDRC_PERF_CNT_FREERUN_CTRL,
952 .cnt_freerun_clr = 0,
953 .cnt_value_wr_op = CN10K_DDRC_PERF_CNT_VALUE_WR_OP,
954 .cnt_value_rd_op = CN10K_DDRC_PERF_CNT_VALUE_RD_OP,
955 .is_cn10k = TRUE,
956 };
957 #endif
958
959 static const struct ddr_pmu_ops ddr_pmu_ody_ops = {
960 .enable_read_freerun_counter = ddr_pmu_ody_enable_read_freerun,
961 .enable_write_freerun_counter = ddr_pmu_ody_enable_write_freerun,
962 .clear_read_freerun_counter = ddr_pmu_ody_read_clear_freerun,
963 .clear_write_freerun_counter = ddr_pmu_ody_write_clear_freerun,
964 .pmu_overflow_handler = ddr_pmu_ody_overflow_hander,
965 };
966
967 #ifdef CONFIG_ACPI
968 static const struct ddr_pmu_platform_data odyssey_ddr_pmu_pdata = {
969 .counter_overflow_val = 0,
970 .counter_max_val = GENMASK_ULL(63, 0),
971 .cnt_base = ODY_DDRC_PERF_CNT_VALUE_BASE,
972 .cfg_base = ODY_DDRC_PERF_CFG_BASE,
973 .cnt_op_mode_ctrl = ODY_DDRC_PERF_CNT_OP_MODE_CTRL,
974 .cnt_start_op_ctrl = ODY_DDRC_PERF_CNT_START_OP_CTRL,
975 .cnt_end_op_ctrl = ODY_DDRC_PERF_CNT_END_OP_CTRL,
976 .cnt_end_status = ODY_DDRC_PERF_CNT_END_STATUS,
977 .cnt_freerun_en = 0,
978 .cnt_freerun_ctrl = ODY_DDRC_PERF_CNT_FREERUN_CTRL,
979 .cnt_freerun_clr = ODY_DDRC_PERF_CNT_FREERUN_CLR,
980 .cnt_value_wr_op = ODY_DDRC_PERF_CNT_VALUE_WR_OP,
981 .cnt_value_rd_op = ODY_DDRC_PERF_CNT_VALUE_RD_OP,
982 .is_ody = TRUE,
983 };
984 #endif
985
cn10k_ddr_perf_probe(struct platform_device * pdev)986 static int cn10k_ddr_perf_probe(struct platform_device *pdev)
987 {
988 const struct ddr_pmu_platform_data *dev_data;
989 struct cn10k_ddr_pmu *ddr_pmu;
990 struct resource *res;
991 void __iomem *base;
992 bool is_cn10k;
993 bool is_ody;
994 char *name;
995 int ret;
996
997 ddr_pmu = devm_kzalloc(&pdev->dev, sizeof(*ddr_pmu), GFP_KERNEL);
998 if (!ddr_pmu)
999 return -ENOMEM;
1000
1001 ddr_pmu->dev = &pdev->dev;
1002 platform_set_drvdata(pdev, ddr_pmu);
1003
1004 dev_data = device_get_match_data(&pdev->dev);
1005 if (!dev_data) {
1006 dev_err(&pdev->dev, "Error: No device match data found\n");
1007 return -ENODEV;
1008 }
1009
1010 base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
1011 if (IS_ERR(base))
1012 return PTR_ERR(base);
1013
1014 ddr_pmu->base = base;
1015
1016 ddr_pmu->p_data = dev_data;
1017 is_cn10k = ddr_pmu->p_data->is_cn10k;
1018 is_ody = ddr_pmu->p_data->is_ody;
1019
1020 if (is_cn10k) {
1021 ddr_pmu->ops = &ddr_pmu_ops;
1022 /* Setup the PMU counter to work in manual mode */
1023 writeq_relaxed(OP_MODE_CTRL_VAL_MANUAL, ddr_pmu->base +
1024 ddr_pmu->p_data->cnt_op_mode_ctrl);
1025
1026 ddr_pmu->pmu = (struct pmu) {
1027 .module = THIS_MODULE,
1028 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
1029 .task_ctx_nr = perf_invalid_context,
1030 .attr_groups = cn10k_attr_groups,
1031 .event_init = cn10k_ddr_perf_event_init,
1032 .add = cn10k_ddr_perf_event_add,
1033 .del = cn10k_ddr_perf_event_del,
1034 .start = cn10k_ddr_perf_event_start,
1035 .stop = cn10k_ddr_perf_event_stop,
1036 .read = cn10k_ddr_perf_event_update,
1037 .pmu_enable = cn10k_ddr_perf_pmu_enable,
1038 .pmu_disable = cn10k_ddr_perf_pmu_disable,
1039 };
1040 }
1041
1042 if (is_ody) {
1043 ddr_pmu->ops = &ddr_pmu_ody_ops;
1044
1045 ddr_pmu->pmu = (struct pmu) {
1046 .module = THIS_MODULE,
1047 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
1048 .task_ctx_nr = perf_invalid_context,
1049 .attr_groups = odyssey_attr_groups,
1050 .event_init = cn10k_ddr_perf_event_init,
1051 .add = cn10k_ddr_perf_event_add,
1052 .del = cn10k_ddr_perf_event_del,
1053 .start = cn10k_ddr_perf_event_start,
1054 .stop = cn10k_ddr_perf_event_stop,
1055 .read = cn10k_ddr_perf_event_update,
1056 };
1057 }
1058
1059 /* Choose this cpu to collect perf data */
1060 ddr_pmu->cpu = raw_smp_processor_id();
1061
1062 name = devm_kasprintf(ddr_pmu->dev, GFP_KERNEL, "mrvl_ddr_pmu_%llx",
1063 res->start);
1064 if (!name)
1065 return -ENOMEM;
1066
1067 hrtimer_init(&ddr_pmu->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1068 ddr_pmu->hrtimer.function = cn10k_ddr_pmu_timer_handler;
1069
1070 cpuhp_state_add_instance_nocalls(
1071 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
1072 &ddr_pmu->node);
1073
1074 ret = perf_pmu_register(&ddr_pmu->pmu, name, -1);
1075 if (ret)
1076 goto error;
1077
1078 pr_info("DDR PMU Driver for ddrc@%llx\n", res->start);
1079 return 0;
1080 error:
1081 cpuhp_state_remove_instance_nocalls(
1082 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
1083 &ddr_pmu->node);
1084 return ret;
1085 }
1086
cn10k_ddr_perf_remove(struct platform_device * pdev)1087 static void cn10k_ddr_perf_remove(struct platform_device *pdev)
1088 {
1089 struct cn10k_ddr_pmu *ddr_pmu = platform_get_drvdata(pdev);
1090
1091 cpuhp_state_remove_instance_nocalls(
1092 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
1093 &ddr_pmu->node);
1094
1095 perf_pmu_unregister(&ddr_pmu->pmu);
1096 }
1097
1098 #ifdef CONFIG_OF
1099 static const struct of_device_id cn10k_ddr_pmu_of_match[] = {
1100 { .compatible = "marvell,cn10k-ddr-pmu", .data = &cn10k_ddr_pmu_pdata },
1101 { },
1102 };
1103 MODULE_DEVICE_TABLE(of, cn10k_ddr_pmu_of_match);
1104 #endif
1105
1106 #ifdef CONFIG_ACPI
1107 static const struct acpi_device_id cn10k_ddr_pmu_acpi_match[] = {
1108 {"MRVL000A", (kernel_ulong_t)&cn10k_ddr_pmu_pdata },
1109 {"MRVL000C", (kernel_ulong_t)&odyssey_ddr_pmu_pdata},
1110 {},
1111 };
1112 MODULE_DEVICE_TABLE(acpi, cn10k_ddr_pmu_acpi_match);
1113 #endif
1114
1115 static struct platform_driver cn10k_ddr_pmu_driver = {
1116 .driver = {
1117 .name = "cn10k-ddr-pmu",
1118 .of_match_table = of_match_ptr(cn10k_ddr_pmu_of_match),
1119 .acpi_match_table = ACPI_PTR(cn10k_ddr_pmu_acpi_match),
1120 .suppress_bind_attrs = true,
1121 },
1122 .probe = cn10k_ddr_perf_probe,
1123 .remove = cn10k_ddr_perf_remove,
1124 };
1125
cn10k_ddr_pmu_init(void)1126 static int __init cn10k_ddr_pmu_init(void)
1127 {
1128 int ret;
1129
1130 ret = cpuhp_setup_state_multi(
1131 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
1132 "perf/marvell/cn10k/ddr:online", NULL,
1133 cn10k_ddr_pmu_offline_cpu);
1134 if (ret)
1135 return ret;
1136
1137 ret = platform_driver_register(&cn10k_ddr_pmu_driver);
1138 if (ret)
1139 cpuhp_remove_multi_state(
1140 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE);
1141 return ret;
1142 }
1143
cn10k_ddr_pmu_exit(void)1144 static void __exit cn10k_ddr_pmu_exit(void)
1145 {
1146 platform_driver_unregister(&cn10k_ddr_pmu_driver);
1147 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE);
1148 }
1149
1150 module_init(cn10k_ddr_pmu_init);
1151 module_exit(cn10k_ddr_pmu_exit);
1152
1153 MODULE_AUTHOR("Bharat Bhushan <[email protected]>");
1154 MODULE_DESCRIPTION("Marvell CN10K DRAM Subsystem (DSS) Performance Monitor Driver");
1155 MODULE_LICENSE("GPL v2");
1156