Lines Matching +full:10 +full:base +full:- +full:te
1 // SPDX-License-Identifier: GPL-2.0
3 * Performance event support for the System z CPU-measurement Sampling Facility
29 #define PERF_EVENT_CPUM_SF 0xB0000UL /* Event: Basic-sampling */
30 #define PERF_EVENT_CPUM_SF_DIAG 0xBD000UL /* Event: Combined-sampling */
31 #define PERF_CPUM_SF_BASIC_MODE 0x0001 /* Basic-sampling flag */
32 #define PERF_CPUM_SF_DIAG_MODE 0x0002 /* Diagnostic-sampling flag */
35 #define OVERFLOW_REG(hwc) ((hwc)->extra_reg.config)
36 #define SFB_ALLOC_REG(hwc) ((hwc)->extra_reg.alloc)
37 #define TEAR_REG(hwc) ((hwc)->last_tag)
38 #define SAMPL_RATE(hwc) ((hwc)->event_base)
39 #define SAMPL_FLAGS(hwc) ((hwc)->config_base)
43 /* Minimum number of sample-data-block-tables:
45 * A single table contains up to 511 pointers to sample-data-blocks.
49 /* Number of sample-data-blocks per sample-data-block-table (SDBT):
50 * A table contains SDB pointers (8 bytes) and one table-link entry
53 #define CPUM_SF_SDB_PER_TABLE ((PAGE_SIZE - 8) / 8)
55 /* Maximum page offset for an SDBT table-link entry:
56 * If this page offset is reached, a table-link entry to the next SDBT
68 * the number of sample-data-block-tables into account. Note that these
69 * numbers apply to the basic-sampling function only.
71 * the diagnostic-sampling function is active.
74 * ---------------------------------------------------
76 * 1 page for SDB-tables
80 * 16 pages for SDB-tables
88 unsigned long *sdbt; /* Sample-data-block-table origin */
90 unsigned long num_sdb; /* Number of sample-data-blocks */
91 unsigned long num_sdbt; /* Number of sample-data-block-tables */
92 unsigned long *tail; /* last sample-data-block-table */
105 /* CPU-measurement sampling information block */
107 /* CPU-measurement sampling control block */
123 return (USEC_PER_SEC / freq) * qsi->cpu_speed; in freq_to_sample_rate()
129 return USEC_PER_SEC * qsi->cpu_speed / rate; in sample_rate_to_freq()
139 ret -= sizeof(struct hws_trailer_entry); in trailer_entry_ptr()
160 * sf_disable() - Switch off sampling facility
171 * sf_buffer_available() - Check for an allocated sampling buffer
175 return !!cpuhw->sfb.sdbt; in sf_buffer_available()
185 sdbt = sfb->sdbt; in free_sampling_buffer()
188 sfb->sdbt = NULL; in free_sampling_buffer()
194 /* Process table-link entries */ in free_sampling_buffer()
209 struct hws_trailer_entry *te; in alloc_sample_data_block() local
212 /* Allocate and initialize sample-data-block */ in alloc_sample_data_block()
215 return -ENOMEM; in alloc_sample_data_block()
216 te = trailer_entry_ptr(sdb); in alloc_sample_data_block()
217 te->header.a = 1; in alloc_sample_data_block()
219 /* Link SDB into the sample-data-block-table */ in alloc_sample_data_block()
226 * realloc_sampling_buffer() - extend sampler memory
228 * Allocates new sample-data-blocks and adds them to the specified sampling
234 * Returns zero on success, non-zero otherwise.
242 if (!sfb->sdbt || !sfb->tail) in realloc_sampling_buffer()
243 return -EINVAL; in realloc_sampling_buffer()
245 if (!is_link_entry(sfb->tail)) in realloc_sampling_buffer()
246 return -EINVAL; in realloc_sampling_buffer()
248 /* Append to the existing sampling buffer, overwriting the table-link in realloc_sampling_buffer()
250 * The tail variables always points to the "tail" (last and table-link) in realloc_sampling_buffer()
251 * entry in an SDB-table. in realloc_sampling_buffer()
253 tail = sfb->tail; in realloc_sampling_buffer()
255 /* Do a sanity check whether the table-link entry points to in realloc_sampling_buffer()
258 if (sfb->sdbt != get_next_sdbt(tail)) { in realloc_sampling_buffer()
260 __func__, (unsigned long)sfb->sdbt, in realloc_sampling_buffer()
262 return -EINVAL; in realloc_sampling_buffer()
268 /* Allocate a new SDB-table if it is full. */ in realloc_sampling_buffer()
272 rc = -ENOMEM; in realloc_sampling_buffer()
275 sfb->num_sdbt++; in realloc_sampling_buffer()
282 /* Allocate a new sample-data-block. in realloc_sampling_buffer()
295 sfb->num_sdbt--; in realloc_sampling_buffer()
301 sfb->num_sdb++; in realloc_sampling_buffer()
307 *tail = virt_to_phys(sfb->sdbt) + 1; in realloc_sampling_buffer()
308 sfb->tail = tail; in realloc_sampling_buffer()
314 * allocate_sampling_buffer() - allocate sampler memory
317 * specified number of sample-data-blocks (SDB). For each allocation,
318 * a 4K page is used. The number of sample-data-block-tables (SDBT)
322 * Returns zero on success, non-zero otherwise.
328 if (sfb->sdbt) in alloc_sampling_buffer()
329 return -EINVAL; in alloc_sampling_buffer()
331 /* Allocate the sample-data-block-table origin */ in alloc_sampling_buffer()
332 sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL); in alloc_sampling_buffer()
333 if (!sfb->sdbt) in alloc_sampling_buffer()
334 return -ENOMEM; in alloc_sampling_buffer()
335 sfb->num_sdb = 0; in alloc_sampling_buffer()
336 sfb->num_sdbt = 1; in alloc_sampling_buffer()
341 sfb->tail = sfb->sdbt; in alloc_sampling_buffer()
342 *sfb->tail = virt_to_phys((void *)sfb->sdbt) + 1; in alloc_sampling_buffer()
344 /* Allocate requested number of sample-data-blocks */ in alloc_sampling_buffer()
372 if (!sfb->sdbt) in sfb_pending_allocs()
374 if (SFB_ALLOC_REG(hwc) > sfb->num_sdb) in sfb_pending_allocs()
375 return SFB_ALLOC_REG(hwc) - sfb->num_sdb; in sfb_pending_allocs()
382 num = min_t(unsigned long, num, sfb_max_limit(hwc) - SFB_ALLOC_REG(hwc)); in sfb_account_allocs()
396 free_sampling_buffer(&cpuhw->sfb); in deallocate_buffers()
412 * Control indicator to trigger a measurement-alert to harvest in allocate_buffers()
413 * sample-data-blocks (SDB). This is done per SDB. This in allocate_buffers()
418 * n-th page. This is counterproductive as one IRQ triggers in allocate_buffers()
428 * 4. Compute the number of sample-data-block-tables (SDBT) and in allocate_buffers()
432 freq = sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc)); in allocate_buffers()
447 return alloc_sampling_buffer(&cpuhw->sfb, in allocate_buffers()
448 sfb_pending_allocs(&cpuhw->sfb, hwc)); in allocate_buffers()
451 static unsigned long min_percent(unsigned int percent, unsigned long base, in min_percent() argument
454 return min_t(unsigned long, min, DIV_ROUND_UP(percent * base, 100)); in min_percent()
457 static unsigned long compute_sfb_extent(unsigned long ratio, unsigned long base) in compute_sfb_extent() argument
459 /* Use a percentage-based approach to extend the sampling facility in compute_sfb_extent()
462 * sample-data-blocks. in compute_sfb_extent()
467 return min_percent(1, base, 1); in compute_sfb_extent()
469 return min_percent(1, base, 1); in compute_sfb_extent()
471 return min_percent(2, base, 2); in compute_sfb_extent()
473 return min_percent(3, base, 3); in compute_sfb_extent()
475 return min_percent(4, base, 4); in compute_sfb_extent()
477 return min_percent(5, base, 8); in compute_sfb_extent()
489 * that has been lost because sample-data-blocks were full. in sfb_account_overflows()
495 ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb, in sfb_account_overflows()
496 sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc))); in sfb_account_overflows()
498 /* Compute number of sample-data-blocks */ in sfb_account_overflows()
499 num = compute_sfb_extent(ratio, cpuhw->sfb.num_sdb); in sfb_account_overflows()
506 /* extend_sampling_buffer() - Extend sampling buffer
534 * new SDB-request cannot be satisfied immediately. in extend_sampling_buffer()
554 qsi(&cpuhw->qsi); in setup_pmc_cpu()
555 cpuhw->flags |= PMU_F_RESERVED; in setup_pmc_cpu()
558 cpuhw->flags &= ~PMU_F_RESERVED; in setup_pmc_cpu()
591 hwc->sample_period = period; in hw_init_period()
592 hwc->last_period = hwc->sample_period; in hw_init_period()
593 local64_set(&hwc->period_left, hwc->sample_period); in hw_init_period()
600 si->min_sampl_rate, si->max_sampl_rate); in hw_limit_rate()
613 pid = -1; in cpumsf_pid_type()
619 if (event->parent) in cpumsf_pid_type()
620 event = event->parent; in cpumsf_pid_type()
621 pid = __task_pid_nr_ns(tsk, type, event->ns); in cpumsf_pid_type()
627 pid = -1; in cpumsf_pid_type()
642 * Obtain the PID from the basic-sampling data entry and in cpumsf_output_event_pid()
643 * correct the data->tid_entry.pid value. in cpumsf_output_event_pid()
645 pid = data->tid_entry.pid; in cpumsf_output_event_pid()
656 data->tid_entry.pid = cpumsf_pid_type(event, pid, PIDTYPE_TGID); in cpumsf_output_event_pid()
657 data->tid_entry.tid = cpumsf_pid_type(event, pid, PIDTYPE_PID); in cpumsf_output_event_pid()
714 struct perf_event_attr *attr = &event->attr; in __hw_perf_event_init_rate()
715 struct hw_perf_event *hwc = &event->hw; in __hw_perf_event_init_rate()
718 if (attr->freq) { in __hw_perf_event_init_rate()
719 if (!attr->sample_freq) in __hw_perf_event_init_rate()
720 return -EINVAL; in __hw_perf_event_init_rate()
721 rate = getrate(attr->freq, attr->sample_freq, si); in __hw_perf_event_init_rate()
722 attr->freq = 0; /* Don't call perf_adjust_period() */ in __hw_perf_event_init_rate()
725 rate = getrate(attr->freq, attr->sample_period, si); in __hw_perf_event_init_rate()
727 return -EINVAL; in __hw_perf_event_init_rate()
729 attr->sample_period = rate; in __hw_perf_event_init_rate()
739 struct perf_event_attr *attr = &event->attr; in __hw_perf_event_init()
740 struct hw_perf_event *hwc = &event->hw; in __hw_perf_event_init()
743 /* Reserve CPU-measurement sampling facility */ in __hw_perf_event_init()
749 event->destroy = hw_perf_event_destroy; in __hw_perf_event_init()
751 /* Access per-CPU sampling information (query sampling info) */ in __hw_perf_event_init()
753 * The event->cpu value can be -1 to count on every CPU, for example, in __hw_perf_event_init()
755 * sampling info from the current CPU, otherwise use event->cpu to in __hw_perf_event_init()
756 * retrieve the per-CPU information. in __hw_perf_event_init()
762 if (event->cpu == -1) { in __hw_perf_event_init()
765 /* Event is pinned to a particular CPU, retrieve the per-CPU in __hw_perf_event_init()
766 * sampling structure for accessing the CPU-specific QSI. in __hw_perf_event_init()
768 cpuhw = &per_cpu(cpu_hw_sf, event->cpu); in __hw_perf_event_init()
769 si = cpuhw->qsi; in __hw_perf_event_init()
777 err = -ENOENT; in __hw_perf_event_init()
783 err = -EBUSY; in __hw_perf_event_init()
793 if (attr->config == PERF_EVENT_CPUM_SF_DIAG) { in __hw_perf_event_init()
795 err = -EPERM; in __hw_perf_event_init()
806 if (attr->config == PERF_EVENT_CPUM_SF_DIAG) in __hw_perf_event_init()
809 /* Allocate the per-CPU sampling buffer using the CPU information in __hw_perf_event_init()
811 * CPU (event->cpu == -1; or cpuhw == NULL), allocate sampling in __hw_perf_event_init()
830 * handler to extract and resolve the PIDs from the basic-sampling in __hw_perf_event_init()
833 if (event->attr.sample_type & PERF_SAMPLE_TID) in __hw_perf_event_init()
835 event->overflow_handler = cpumsf_output_event_pid; in __hw_perf_event_init()
843 u64 sample_type = event->attr.sample_type; in is_callchain_event()
856 return -EOPNOTSUPP; in cpumsf_pmu_event_init()
858 switch (event->attr.type) { in cpumsf_pmu_event_init()
860 if ((event->attr.config != PERF_EVENT_CPUM_SF) && in cpumsf_pmu_event_init()
861 (event->attr.config != PERF_EVENT_CPUM_SF_DIAG)) in cpumsf_pmu_event_init()
862 return -ENOENT; in cpumsf_pmu_event_init()
870 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES) in cpumsf_pmu_event_init()
871 return -ENOENT; in cpumsf_pmu_event_init()
873 return -ENOENT; in cpumsf_pmu_event_init()
876 return -ENOENT; in cpumsf_pmu_event_init()
882 if (event->attr.exclude_hv) in cpumsf_pmu_event_init()
883 event->attr.exclude_hv = 0; in cpumsf_pmu_event_init()
884 if (event->attr.exclude_idle) in cpumsf_pmu_event_init()
885 event->attr.exclude_idle = 0; in cpumsf_pmu_event_init()
899 * - added/started on this CPU (PMU_F_IN_USE set) in cpumsf_pmu_enable()
900 * - and CPU must be available (PMU_F_RESERVED set) in cpumsf_pmu_enable()
901 * - and not already enabled (PMU_F_ENABLED not set) in cpumsf_pmu_enable()
902 * - and not in error condition (PMU_F_ERR_MASK not set) in cpumsf_pmu_enable()
904 if (cpuhw->flags != (PMU_F_IN_USE | PMU_F_RESERVED)) in cpumsf_pmu_enable()
915 * facility, but it can be fully re-enabled using sampling controls that in cpumsf_pmu_enable()
918 hwc = &cpuhw->event->hw; in cpumsf_pmu_enable()
921 * Account number of overflow-designated buffer extents in cpumsf_pmu_enable()
924 extend_sampling_buffer(&cpuhw->sfb, hwc); in cpumsf_pmu_enable()
927 cpuhw->lsctl.interval = SAMPL_RATE(hwc); in cpumsf_pmu_enable()
930 err = lsctl(&cpuhw->lsctl); in cpumsf_pmu_enable()
937 lpp(&get_lowcore()->lpp); in cpumsf_pmu_enable()
938 cpuhw->flags |= PMU_F_ENABLED; in cpumsf_pmu_enable()
948 if (!(cpuhw->flags & PMU_F_ENABLED)) in cpumsf_pmu_disable()
951 if (cpuhw->flags & PMU_F_ERR_MASK) in cpumsf_pmu_disable()
955 inactive = cpuhw->lsctl; in cpumsf_pmu_disable()
974 cpuhw->lsctl.tear = si.tear; in cpumsf_pmu_disable()
975 cpuhw->lsctl.dear = si.dear; in cpumsf_pmu_disable()
978 cpuhw->flags &= ~PMU_F_ENABLED; in cpumsf_pmu_disable()
981 /* perf_event_exclude() - Filter event
984 * @sde_regs: Sample-data-entry (sde) regs structure
988 * Return non-zero if the event shall be excluded.
993 if (event->attr.exclude_user && user_mode(regs)) in perf_event_exclude()
995 if (event->attr.exclude_kernel && !user_mode(regs)) in perf_event_exclude()
997 if (event->attr.exclude_guest && sde_regs->in_guest) in perf_event_exclude()
999 if (event->attr.exclude_host && !sde_regs->in_guest) in perf_event_exclude()
1004 /* perf_push_sample() - Push samples to perf
1013 * Return non-zero if an event overflow occurred.
1024 perf_sample_data_init(&data, 0, event->hw.last_period); in perf_push_sample()
1026 /* Setup pt_regs to look like an CPU-measurement external interrupt in perf_push_sample()
1028 * field which is unused contains additional sample-data-entry related in perf_push_sample()
1036 psw_bits(regs.psw).ia = basic->ia; in perf_push_sample()
1037 psw_bits(regs.psw).dat = basic->T; in perf_push_sample()
1038 psw_bits(regs.psw).wait = basic->W; in perf_push_sample()
1039 psw_bits(regs.psw).pstate = basic->P; in perf_push_sample()
1040 psw_bits(regs.psw).as = basic->AS; in perf_push_sample()
1046 * A non-zero guest program parameter always indicates a guest in perf_push_sample()
1053 switch (basic->CL) { in perf_push_sample()
1055 sde_regs->in_guest = 0; in perf_push_sample()
1058 sde_regs->in_guest = 1; in perf_push_sample()
1061 if (basic->gpp || basic->prim_asn != 0xffff) in perf_push_sample()
1062 sde_regs->in_guest = 1; in perf_push_sample()
1067 * Store the PID value from the sample-data-entry to be in perf_push_sample()
1070 data.tid_entry.pid = basic->hpp & LPP_PID_MASK; in perf_push_sample()
1077 event->pmu->stop(event, 0); in perf_push_sample()
1086 local64_add(count, &event->count); in perf_event_count_update()
1089 /* hw_collect_samples() - Walk through a sample-data-block and collect samples
1091 * @sdbt: Sample-data-block table
1094 * Walks through a sample-data-block and collects sampling data entries that are
1096 * there can be either basic-sampling or combined-sampling data entries. A
1097 * combined-sampling data entry consists of a basic- and a diagnostic-sampling
1099 * event hardware structure. The function always works with a combined-sampling
1102 * Note that the implementation focuses on basic-sampling data entries and, if
1103 * such an entry is not valid, the entire combined-sampling data entry is
1112 struct hws_trailer_entry *te; in hw_collect_samples() local
1115 te = trailer_entry_ptr((unsigned long)sdbt); in hw_collect_samples()
1117 while ((unsigned long *)sample < (unsigned long *)te) { in hw_collect_samples()
1119 if (!sample->def || sample->LS) in hw_collect_samples()
1123 perf_event_count_update(event, SAMPL_RATE(&event->hw)); in hw_collect_samples()
1126 if (sample->def == 0x0001) { in hw_collect_samples()
1133 if (sample->I == 0 && sample->W == 0) { in hw_collect_samples()
1145 * from a combined basic- and diagnostic-sampling. in hw_collect_samples()
1146 * If only basic-sampling is then active, entries are in hw_collect_samples()
1148 * This is typically the case for sample-data-blocks in hw_collect_samples()
1152 if (!te->header.f) in hw_collect_samples()
1157 sample->def = 0; in hw_collect_samples()
1162 /* hw_perf_event_update() - Process sampling buffer
1164 * @flush_all: Flag to also flush partially filled sample-data-blocks
1170 * Only full sample-data-blocks are processed. Specify the flush_all flag
1171 * to also walk through partially filled sample-data-blocks.
1176 struct hw_perf_event *hwc = &event->hw; in hw_perf_event_update()
1178 struct hws_trailer_entry *te; in hw_perf_event_update() local
1192 /* Get the trailer entry of the sample-data-block */ in hw_perf_event_update()
1194 te = trailer_entry_ptr(sdb); in hw_perf_event_update()
1197 if (!te->header.f) { in hw_perf_event_update()
1204 if (te->header.overflow) in hw_perf_event_update()
1209 sampl_overflow += te->header.overflow; in hw_perf_event_update()
1211 /* Collect all samples from a single sample-data-block and in hw_perf_event_update()
1218 /* Reset trailer (using compare-double-and-swap) */ in hw_perf_event_update()
1219 prev.val = READ_ONCE_ALIGNED_128(te->header.val); in hw_perf_event_update()
1225 } while (!try_cmpxchg128(&te->header.val, &prev.val, new.val)); in hw_perf_event_update()
1227 /* Advance to next sample-data-block */ in hw_perf_event_update()
1235 /* Stop processing sample-data if all samples of the current in hw_perf_event_update()
1236 * sample-data-block were flushed even if it was not full. in hw_perf_event_update()
1256 SAMPL_RATE(hwc) += DIV_ROUND_UP(SAMPL_RATE(hwc), 10); in hw_perf_event_update()
1262 return i % aux->sfb.num_sdb; in aux_sdb_index()
1267 return end >= start ? end - start + 1 : 0; in aux_sdb_num()
1272 return aux_sdb_num(aux->head, aux->alert_mark); in aux_sdb_num_alert()
1277 return aux_sdb_num(aux->head, aux->empty_mark); in aux_sdb_num_empty()
1289 sdb = aux->sdb_index[index]; in aux_sdb_trailer()
1306 struct hws_trailer_entry *te; in aux_output_end() local
1313 for (i = 0, idx = aux->head; i < range_scan; i++, idx++) { in aux_output_end()
1314 te = aux_sdb_trailer(aux, idx); in aux_output_end()
1315 if (!te->header.f) in aux_output_end()
1322 te = aux_sdb_trailer(aux, aux->alert_mark); in aux_output_end()
1323 te->header.a = 0; in aux_output_end()
1339 unsigned long range, i, range_scan, idx, head, base, offset; in aux_output_begin() local
1340 struct hws_trailer_entry *te; in aux_output_begin() local
1342 if (handle->head & ~PAGE_MASK) in aux_output_begin()
1343 return -EINVAL; in aux_output_begin()
1345 aux->head = handle->head >> PAGE_SHIFT; in aux_output_begin()
1346 range = (handle->size + 1) >> PAGE_SHIFT; in aux_output_begin()
1348 return -ENOMEM; in aux_output_begin()
1351 * SDBs between aux->head and aux->empty_mark are already ready in aux_output_begin()
1355 range_scan = range - aux_sdb_num_empty(aux); in aux_output_begin()
1356 idx = aux->empty_mark + 1; in aux_output_begin()
1358 te = aux_sdb_trailer(aux, idx); in aux_output_begin()
1359 te->header.f = 0; in aux_output_begin()
1360 te->header.a = 0; in aux_output_begin()
1361 te->header.overflow = 0; in aux_output_begin()
1364 aux->empty_mark = aux->head + range - 1; in aux_output_begin()
1368 aux->alert_mark = aux->head + range/2 - 1; in aux_output_begin()
1369 te = aux_sdb_trailer(aux, aux->alert_mark); in aux_output_begin()
1370 te->header.a = 1; in aux_output_begin()
1373 head = aux_sdb_index(aux, aux->head); in aux_output_begin()
1374 base = aux->sdbt_index[head / CPUM_SF_SDB_PER_TABLE]; in aux_output_begin()
1376 cpuhw->lsctl.tear = virt_to_phys((void *)base) + offset * sizeof(unsigned long); in aux_output_begin()
1377 cpuhw->lsctl.dear = virt_to_phys((void *)aux->sdb_index[head]); in aux_output_begin()
1392 struct hws_trailer_entry *te; in aux_set_alert() local
1394 te = aux_sdb_trailer(aux, alert_index); in aux_set_alert()
1395 prev.val = READ_ONCE_ALIGNED_128(te->header.val); in aux_set_alert()
1409 } while (!try_cmpxchg128(&te->header.val, &prev.val, new.val)); in aux_set_alert()
1414 * aux_reset_buffer() - Scan and setup SDBs for new samples
1416 * @range: The range of SDBs to scan started from aux->head
1419 * Set alert indicator on the SDB at index of aux->alert_mark. If this SDB is
1424 * Scan the SDBs in AUX buffer from behind aux->empty_mark. They are used
1427 * If aux->alert_mark fall in this area, just set it. Overflow count is
1430 * SDBs between aux->head and aux->empty_mark are already reset at last time.
1441 struct hws_trailer_entry *te; in aux_reset_buffer() local
1449 return aux_set_alert(aux, aux->alert_mark, overflow); in aux_reset_buffer()
1451 if (aux->alert_mark <= aux->empty_mark) in aux_reset_buffer()
1456 if (!aux_set_alert(aux, aux->alert_mark, overflow)) in aux_reset_buffer()
1464 range_scan = range - aux_sdb_num_empty(aux); in aux_reset_buffer()
1465 idx = aux->empty_mark + 1; in aux_reset_buffer()
1467 te = aux_sdb_trailer(aux, idx); in aux_reset_buffer()
1468 prev.val = READ_ONCE_ALIGNED_128(te->header.val); in aux_reset_buffer()
1474 if (idx == aux->alert_mark) in aux_reset_buffer()
1478 } while (!try_cmpxchg128(&te->header.val, &prev.val, new.val)); in aux_reset_buffer()
1483 aux->empty_mark = aux->head + range - 1; in aux_reset_buffer()
1497 struct perf_output_handle *handle = &cpuhw->handle; in hw_collect_aux()
1510 num_sdb = aux->sfb.num_sdb; in hw_collect_aux()
1513 aux = perf_aux_output_begin(handle, cpuhw->event); in hw_collect_aux()
1514 if (handle->size == 0) { in hw_collect_aux()
1516 "diagnostic-sampling mode is full\n", in hw_collect_aux()
1524 aux->head = handle->head >> PAGE_SHIFT; in hw_collect_aux()
1525 range = (handle->size + 1) >> PAGE_SHIFT; in hw_collect_aux()
1527 aux->alert_mark = aux->head; in hw_collect_aux()
1529 aux->alert_mark = aux->head + range/2 - 1; in hw_collect_aux()
1537 perf_aux_output_end(&cpuhw->handle, size); in hw_collect_aux()
1539 "pages to overflow\n", aux->sfb.num_sdb); in hw_collect_aux()
1542 perf_aux_output_end(&cpuhw->handle, size); in hw_collect_aux()
1559 num_sdbt = aux->sfb.num_sdbt; in aux_buffer_free()
1561 free_page(aux->sdbt_index[i]); in aux_buffer_free()
1563 kfree(aux->sdbt_index); in aux_buffer_free()
1564 kfree(aux->sdb_index); in aux_buffer_free()
1570 struct hws_trailer_entry *te; in aux_sdb_init() local
1572 te = trailer_entry_ptr(sdb); in aux_sdb_init()
1574 /* Save clock base */ in aux_sdb_init()
1575 te->clock_base = 1; in aux_sdb_init()
1576 te->progusage2 = tod_clock_base.tod; in aux_sdb_init()
1580 * aux_buffer_setup() - Setup AUX buffer for diagnostic mode sampling
1581 * @event: Event the buffer is setup for, event->cpu == -1 means current
1589 * the task among online cpus when it is a per-thread event.
1620 sfb = &aux->sfb; in aux_buffer_setup()
1624 aux->sdbt_index = kmalloc_array(n_sdbt, sizeof(void *), GFP_KERNEL); in aux_buffer_setup()
1625 if (!aux->sdbt_index) in aux_buffer_setup()
1629 aux->sdb_index = kmalloc_array(nr_pages, sizeof(void *), GFP_KERNEL); in aux_buffer_setup()
1630 if (!aux->sdb_index) in aux_buffer_setup()
1634 sfb->num_sdbt = 0; in aux_buffer_setup()
1635 sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL); in aux_buffer_setup()
1636 if (!sfb->sdbt) in aux_buffer_setup()
1638 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt; in aux_buffer_setup()
1639 tail = sfb->tail = sfb->sdbt; in aux_buffer_setup()
1650 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new; in aux_buffer_setup()
1657 aux->sdb_index[i] = (unsigned long)pages[i]; in aux_buffer_setup()
1660 sfb->num_sdb = nr_pages; in aux_buffer_setup()
1663 *tail = virt_to_phys(sfb->sdbt) + 1; in aux_buffer_setup()
1664 sfb->tail = tail; in aux_buffer_setup()
1671 aux->empty_mark = sfb->num_sdb - 1; in aux_buffer_setup()
1677 for (i = 0; i < sfb->num_sdbt; i++) in aux_buffer_setup()
1678 free_page(aux->sdbt_index[i]); in aux_buffer_setup()
1679 kfree(aux->sdb_index); in aux_buffer_setup()
1681 kfree(aux->sdbt_index); in aux_buffer_setup()
1690 /* Nothing to do ... updates are interrupt-driven */ in cpumsf_pmu_read()
1695 * Return non-zero on error and zero on passed checks.
1704 if (event->cpu == -1) { in cpumsf_pmu_check_period()
1707 /* Event is pinned to a particular CPU, retrieve the per-CPU in cpumsf_pmu_check_period()
1708 * sampling structure for accessing the CPU-specific QSI. in cpumsf_pmu_check_period()
1710 struct cpu_hw_sf *cpuhw = &per_cpu(cpu_hw_sf, event->cpu); in cpumsf_pmu_check_period()
1712 si = cpuhw->qsi; in cpumsf_pmu_check_period()
1715 do_freq = !!SAMPL_FREQ_MODE(&event->hw); in cpumsf_pmu_check_period()
1718 return -EINVAL; in cpumsf_pmu_check_period()
1720 event->attr.sample_period = rate; in cpumsf_pmu_check_period()
1721 SAMPL_RATE(&event->hw) = rate; in cpumsf_pmu_check_period()
1722 hw_init_period(&event->hw, SAMPL_RATE(&event->hw)); in cpumsf_pmu_check_period()
1733 if (!(event->hw.state & PERF_HES_STOPPED)) in cpumsf_pmu_start()
1735 perf_pmu_disable(event->pmu); in cpumsf_pmu_start()
1736 event->hw.state = 0; in cpumsf_pmu_start()
1737 cpuhw->lsctl.cs = 1; in cpumsf_pmu_start()
1738 if (SAMPL_DIAG_MODE(&event->hw)) in cpumsf_pmu_start()
1739 cpuhw->lsctl.cd = 1; in cpumsf_pmu_start()
1740 perf_pmu_enable(event->pmu); in cpumsf_pmu_start()
1750 if (event->hw.state & PERF_HES_STOPPED) in cpumsf_pmu_stop()
1753 perf_pmu_disable(event->pmu); in cpumsf_pmu_stop()
1754 cpuhw->lsctl.cs = 0; in cpumsf_pmu_stop()
1755 cpuhw->lsctl.cd = 0; in cpumsf_pmu_stop()
1756 event->hw.state |= PERF_HES_STOPPED; in cpumsf_pmu_stop()
1758 if ((flags & PERF_EF_UPDATE) && !(event->hw.state & PERF_HES_UPTODATE)) { in cpumsf_pmu_stop()
1760 if (cpuhw->flags & PMU_F_RESERVED) in cpumsf_pmu_stop()
1762 event->hw.state |= PERF_HES_UPTODATE; in cpumsf_pmu_stop()
1764 perf_pmu_enable(event->pmu); in cpumsf_pmu_stop()
1773 if (cpuhw->flags & PMU_F_IN_USE) in cpumsf_pmu_add()
1774 return -EAGAIN; in cpumsf_pmu_add()
1776 if (!SAMPL_DIAG_MODE(&event->hw) && !sf_buffer_available(cpuhw)) in cpumsf_pmu_add()
1777 return -EINVAL; in cpumsf_pmu_add()
1779 perf_pmu_disable(event->pmu); in cpumsf_pmu_add()
1781 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; in cpumsf_pmu_add()
1784 * using the SDB-table start. Reset TEAR_REG event hardware register in cpumsf_pmu_add()
1788 cpuhw->lsctl.s = 0; in cpumsf_pmu_add()
1789 cpuhw->lsctl.h = 1; in cpumsf_pmu_add()
1790 cpuhw->lsctl.interval = SAMPL_RATE(&event->hw); in cpumsf_pmu_add()
1791 if (!SAMPL_DIAG_MODE(&event->hw)) { in cpumsf_pmu_add()
1792 cpuhw->lsctl.tear = virt_to_phys(cpuhw->sfb.sdbt); in cpumsf_pmu_add()
1793 cpuhw->lsctl.dear = *(unsigned long *)cpuhw->sfb.sdbt; in cpumsf_pmu_add()
1794 TEAR_REG(&event->hw) = (unsigned long)cpuhw->sfb.sdbt; in cpumsf_pmu_add()
1799 if (WARN_ON_ONCE(cpuhw->lsctl.es == 1 || cpuhw->lsctl.ed == 1)) { in cpumsf_pmu_add()
1800 err = -EAGAIN; in cpumsf_pmu_add()
1803 if (SAMPL_DIAG_MODE(&event->hw)) { in cpumsf_pmu_add()
1804 aux = perf_aux_output_begin(&cpuhw->handle, event); in cpumsf_pmu_add()
1806 err = -EINVAL; in cpumsf_pmu_add()
1809 err = aux_output_begin(&cpuhw->handle, aux, cpuhw); in cpumsf_pmu_add()
1812 cpuhw->lsctl.ed = 1; in cpumsf_pmu_add()
1814 cpuhw->lsctl.es = 1; in cpumsf_pmu_add()
1817 cpuhw->event = event; in cpumsf_pmu_add()
1818 cpuhw->flags |= PMU_F_IN_USE; in cpumsf_pmu_add()
1824 perf_pmu_enable(event->pmu); in cpumsf_pmu_add()
1832 perf_pmu_disable(event->pmu); in cpumsf_pmu_del()
1835 cpuhw->lsctl.es = 0; in cpumsf_pmu_del()
1836 cpuhw->lsctl.ed = 0; in cpumsf_pmu_del()
1837 cpuhw->flags &= ~PMU_F_IN_USE; in cpumsf_pmu_del()
1838 cpuhw->event = NULL; in cpumsf_pmu_del()
1840 if (SAMPL_DIAG_MODE(&event->hw)) in cpumsf_pmu_del()
1841 aux_output_end(&cpuhw->handle); in cpumsf_pmu_del()
1843 perf_pmu_enable(event->pmu); in cpumsf_pmu_del()
1875 PMU_FORMAT_ATTR(event, "config:0-63");
1930 if (!(cpuhw->flags & PMU_F_RESERVED)) in cpumf_measurement_alert()
1938 if (cpuhw->flags & PMU_F_IN_USE) { in cpumf_measurement_alert()
1939 if (SAMPL_DIAG_MODE(&cpuhw->event->hw)) in cpumf_measurement_alert()
1942 hw_perf_event_update(cpuhw->event, 0); in cpumf_measurement_alert()
1946 /* Report measurement alerts only for non-PRA codes */ in cpumf_measurement_alert()
1953 qsi(&cpuhw->qsi); in cpumf_measurement_alert()
1955 /* Loss of sample data due to high-priority machine activities */ in cpumf_measurement_alert()
1958 cpuhw->flags |= PMU_F_ERR_LSDA; in cpumf_measurement_alert()
1966 cpuhw->flags |= PMU_F_ERR_IBE; in cpumf_measurement_alert()
1998 return -ENODEV; in param_get_sfb_size()
2008 return -ENODEV; in param_set_sfb_size()
2010 return -EINVAL; in param_set_sfb_size()
2016 rc = (sscanf(val, "%lu,%lu", &min, &max) == 2) ? 0 : -EINVAL; in param_set_sfb_size()
2018 rc = kstrtoul(val, 10, &max); in param_set_sfb_size()
2021 rc = -EINVAL; in param_set_sfb_size()
2056 return -ENODEV; in init_cpum_sampling_pmu()
2061 return -ENODEV; in init_cpum_sampling_pmu()
2065 return -EINVAL; in init_cpum_sampling_pmu()
2080 return -ENOMEM; in init_cpum_sampling_pmu()