Lines Matching +full:y +full:- +full:rp
1 // SPDX-License-Identifier: GPL-2.0-only
29 #include <asm/intel-family.h>
105 #define NR_RAW_PRIMITIVES (NR_RAPL_PRIMITIVES - 2)
129 return rd->rpl[pl].name ? true : false; in is_pl_valid()
134 if (rd->rp->priv->type == RAPL_IF_TPMI) { in get_pl_lock_prim()
145 return -EINVAL; in get_pl_lock_prim()
151 if (rd->rp->priv->limits[rd->id] & BIT(POWER_LIMIT2)) in get_pl_lock_prim()
162 if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI) in get_pl_prim()
172 return -EINVAL; in get_pl_prim()
176 if (prim == PL_CLAMP && rd->rp->priv->type != RAPL_IF_TPMI) in get_pl_prim()
186 return -EINVAL; in get_pl_prim()
197 return -EINVAL; in get_pl_prim()
199 return -EINVAL; in get_pl_prim()
219 static struct rapl_defaults *get_defaults(struct rapl_package *rp) in get_defaults() argument
221 return rp->priv->defaults; in get_defaults()
252 static void rapl_init_domains(struct rapl_package *rp);
267 static void package_power_limit_irq_save(struct rapl_package *rp);
299 return -EIO; in get_energy_counter()
313 struct rapl_package *rp = rd->rp; in release_zone() local
318 if (rd->id == RAPL_DOMAIN_PACKAGE) { in release_zone()
320 rp->domains = NULL; in release_zone()
342 struct rapl_defaults *defaults = get_defaults(rd->rp); in set_domain_enable()
347 if (!ret && defaults->set_floor_freq) in set_domain_enable()
348 defaults->set_floor_freq(rd, mode); in set_domain_enable()
360 if (rd->rpl[POWER_LIMIT1].locked) { in get_domain_enable()
419 * index in that some PLs maybe missing due to non-existent MSRs. So we
434 return -EINVAL; in contraint_to_pl()
441 struct rapl_package *rp; in set_power_limit() local
448 rp = rd->rp; in set_power_limit()
452 package_power_limit_irq_save(rp); in set_power_limit()
525 return rd->rpl[id].name; in get_constraint_name()
564 static int get_rid(struct rapl_package *rp) in get_rid() argument
566 return rp->lead_cpu >= 0 ? rp->lead_cpu : rp->id; in get_rid()
570 static void rapl_init_domains(struct rapl_package *rp) in rapl_init_domains() argument
574 struct rapl_domain *rd = rp->domains; in rapl_init_domains()
577 unsigned int mask = rp->domain_map & (1 << i); in rapl_init_domains()
583 rd->rp = rp; in rapl_init_domains()
585 if (i == RAPL_DOMAIN_PLATFORM && rp->id > 0) { in rapl_init_domains()
586 snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "psys-%d", in rapl_init_domains()
587 rp->lead_cpu >= 0 ? topology_physical_package_id(rp->lead_cpu) : in rapl_init_domains()
588 rp->id); in rapl_init_domains()
590 snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "%s", in rapl_init_domains()
594 rd->id = i; in rapl_init_domains()
597 rp->priv->limits[i] |= BIT(POWER_LIMIT1); in rapl_init_domains()
600 if (rp->priv->limits[i] & BIT(t)) in rapl_init_domains()
601 rd->rpl[t].name = pl_names[t]; in rapl_init_domains()
605 rd->regs[j] = rp->priv->regs[i][j]; in rapl_init_domains()
615 struct rapl_defaults *defaults = get_defaults(rd->rp); in rapl_unit_xlate()
620 units = rd->power_unit; in rapl_unit_xlate()
624 units = rd->energy_unit; in rapl_unit_xlate()
627 return defaults->compute_time_window(rd, value, to_raw); in rapl_unit_xlate()
692 /* non-hardware */
734 /* non-hardware */
739 static struct rapl_primitive_info *get_rpi(struct rapl_package *rp, int prim) in get_rpi() argument
741 struct rapl_primitive_info *rpi = rp->priv->rpi; in get_rpi()
749 static int rapl_config(struct rapl_package *rp) in rapl_config() argument
751 switch (rp->priv->type) { in rapl_config()
755 rp->priv->defaults = (void *)defaults_msr; in rapl_config()
756 rp->priv->rpi = (void *)rpi_msr; in rapl_config()
759 rp->priv->defaults = (void *)&defaults_tpmi; in rapl_config()
760 rp->priv->rpi = (void *)rpi_tpmi; in rapl_config()
763 return -EINVAL; in rapl_config()
767 if (!rp->priv->defaults || !rp->priv->rpi) in rapl_config()
768 return -ENODEV; in rapl_config()
776 struct rapl_defaults *defaults = get_defaults(rd->rp); in prim_fixups()
778 if (!defaults->spr_psys_bits) in prim_fixups()
781 if (rd->id != RAPL_DOMAIN_PLATFORM) in prim_fixups()
805 * RAPL MSRs are non-architectual and are laid out not consistently across
809 * is pre-assigned based on RAPL unit MSRs read at init time.
810 * 63-------------------------- 31--------------------------- 0
812 * | |<- shift ----------------|
813 * 63-------------------------- 31--------------------------- 0
820 struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed); in rapl_read_data_raw()
823 if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY) in rapl_read_data_raw()
824 return -EINVAL; in rapl_read_data_raw()
826 ra.reg = rd->regs[rpi->id]; in rapl_read_data_raw()
828 return -EINVAL; in rapl_read_data_raw()
830 /* non-hardware data are collected by the polling thread */ in rapl_read_data_raw()
831 if (rpi->flag & RAPL_PRIMITIVE_DERIVED) { in rapl_read_data_raw()
832 *data = rd->rdd.primitives[prim]; in rapl_read_data_raw()
836 ra.mask = rpi->mask; in rapl_read_data_raw()
838 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { in rapl_read_data_raw()
839 pr_debug("failed to read reg 0x%llx for %s:%s\n", ra.reg.val, rd->rp->name, rd->name); in rapl_read_data_raw()
840 return -EIO; in rapl_read_data_raw()
843 value = ra.value >> rpi->shift; in rapl_read_data_raw()
846 *data = rapl_unit_xlate(rd, rpi->unit, value, 0); in rapl_read_data_raw()
859 struct rapl_primitive_info *rpi = get_rpi(rd->rp, prim_fixed); in rapl_write_data_raw()
864 if (!rpi || !rpi->name || rpi->flag & RAPL_PRIMITIVE_DUMMY) in rapl_write_data_raw()
865 return -EINVAL; in rapl_write_data_raw()
867 bits = rapl_unit_xlate(rd, rpi->unit, value, 1); in rapl_write_data_raw()
868 bits <<= rpi->shift; in rapl_write_data_raw()
869 bits &= rpi->mask; in rapl_write_data_raw()
873 ra.reg = rd->regs[rpi->id]; in rapl_write_data_raw()
874 ra.mask = rpi->mask; in rapl_write_data_raw()
877 ret = rd->rp->priv->write_raw(get_rid(rd->rp), &ra); in rapl_write_data_raw()
888 return -EINVAL; in rapl_read_pl_data()
900 return -EINVAL; in rapl_write_pl_data()
902 if (rd->rpl[pl].locked) { in rapl_write_pl_data()
903 pr_debug("%s:%s:%s locked by BIOS\n", rd->rp->name, rd->name, pl_names[pl]); in rapl_write_pl_data()
904 return -EACCES; in rapl_write_pl_data()
925 ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; in rapl_check_unit_core()
927 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { in rapl_check_unit_core()
929 ra.reg.val, rd->rp->name, rd->name); in rapl_check_unit_core()
930 return -ENODEV; in rapl_check_unit_core()
934 rd->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value); in rapl_check_unit_core()
937 rd->power_unit = 1000000 / (1 << value); in rapl_check_unit_core()
940 rd->time_unit = 1000000 / (1 << value); in rapl_check_unit_core()
943 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit); in rapl_check_unit_core()
953 ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; in rapl_check_unit_atom()
955 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { in rapl_check_unit_atom()
957 ra.reg.val, rd->rp->name, rd->name); in rapl_check_unit_atom()
958 return -ENODEV; in rapl_check_unit_atom()
962 rd->energy_unit = ENERGY_UNIT_SCALE * 1 << value; in rapl_check_unit_atom()
965 rd->power_unit = (1 << value) * 1000; in rapl_check_unit_atom()
968 rd->time_unit = 1000000 / (1 << value); in rapl_check_unit_atom()
971 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit); in rapl_check_unit_atom()
979 struct rapl_package *rp = (struct rapl_package *)info; in power_limit_irq_save_cpu() local
983 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) { in power_limit_irq_save_cpu()
984 rp->power_limit_irq = l & PACKAGE_THERM_INT_PLN_ENABLE; in power_limit_irq_save_cpu()
985 rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED; in power_limit_irq_save_cpu()
1001 static void package_power_limit_irq_save(struct rapl_package *rp) in package_power_limit_irq_save() argument
1003 if (rp->lead_cpu < 0) in package_power_limit_irq_save()
1009 smp_call_function_single(rp->lead_cpu, power_limit_irq_save_cpu, rp, 1); in package_power_limit_irq_save()
1016 static void package_power_limit_irq_restore(struct rapl_package *rp) in package_power_limit_irq_restore() argument
1020 if (rp->lead_cpu < 0) in package_power_limit_irq_restore()
1027 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) in package_power_limit_irq_restore()
1032 if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE) in package_power_limit_irq_restore()
1044 /* always enable clamp such that p-state can go below OS requested in set_floor_freq_default()
1058 struct rapl_defaults *defaults = get_defaults(rd->rp); in set_floor_freq_atom()
1061 if (!defaults->floor_freq_reg_addr) { in set_floor_freq_atom()
1068 defaults->floor_freq_reg_addr, in set_floor_freq_atom()
1076 defaults->floor_freq_reg_addr, mdata); in set_floor_freq_atom()
1082 u64 f, y; /* fraction and exp. used for time unit */ in rapl_compute_time_window_core() local
1085 * Special processing based on 2^Y*(1+F/4), refer in rapl_compute_time_window_core()
1090 y = value & 0x1f; in rapl_compute_time_window_core()
1091 value = (1 << y) * (4 + f) * rd->time_unit / 4; in rapl_compute_time_window_core()
1093 if (value < rd->time_unit) in rapl_compute_time_window_core()
1096 do_div(value, rd->time_unit); in rapl_compute_time_window_core()
1097 y = ilog2(value); in rapl_compute_time_window_core()
1103 if (y > 0x1f) in rapl_compute_time_window_core()
1106 f = div64_u64(4 * (value - (1ULL << y)), 1ULL << y); in rapl_compute_time_window_core()
1107 value = (y & 0x1f) | ((f & 0x3) << 5); in rapl_compute_time_window_core()
1120 return (value) ? value * rd->time_unit : rd->time_unit; in rapl_compute_time_window_atom()
1122 value = div64_u64(value, rd->time_unit); in rapl_compute_time_window_atom()
1140 ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; in rapl_check_unit_tpmi()
1142 if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { in rapl_check_unit_tpmi()
1144 ra.reg.val, rd->rp->name, rd->name); in rapl_check_unit_tpmi()
1145 return -ENODEV; in rapl_check_unit_tpmi()
1149 rd->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value); in rapl_check_unit_tpmi()
1152 rd->power_unit = 1000000 / (1 << value); in rapl_check_unit_tpmi()
1155 rd->time_unit = 1000000 / (1 << value); in rapl_check_unit_tpmi()
1158 rd->rp->name, rd->name, rd->energy_unit, rd->time_unit, rd->power_unit); in rapl_check_unit_tpmi()
1297 static void rapl_update_domain_data(struct rapl_package *rp) in rapl_update_domain_data() argument
1302 for (dmn = 0; dmn < rp->nr_domains; dmn++) { in rapl_update_domain_data()
1303 pr_debug("update %s domain %s data\n", rp->name, in rapl_update_domain_data()
1304 rp->domains[dmn].name); in rapl_update_domain_data()
1305 /* exclude non-raw primitives */ in rapl_update_domain_data()
1307 struct rapl_primitive_info *rpi = get_rpi(rp, prim); in rapl_update_domain_data()
1309 if (!rapl_read_data_raw(&rp->domains[dmn], prim, in rapl_update_domain_data()
1310 rpi->unit, &val)) in rapl_update_domain_data()
1311 rp->domains[dmn].rdd.primitives[prim] = val; in rapl_update_domain_data()
1317 static int rapl_package_register_powercap(struct rapl_package *rp) in rapl_package_register_powercap() argument
1324 rapl_update_domain_data(rp); in rapl_package_register_powercap()
1327 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { in rapl_package_register_powercap()
1328 if (rd->id == RAPL_DOMAIN_PACKAGE) { in rapl_package_register_powercap()
1330 pr_debug("register package domain %s\n", rp->name); in rapl_package_register_powercap()
1331 power_zone = powercap_register_zone(&rd->power_zone, in rapl_package_register_powercap()
1332 rp->priv->control_type, rp->name, in rapl_package_register_powercap()
1333 NULL, &zone_ops[rd->id], nr_pl, in rapl_package_register_powercap()
1337 rp->name); in rapl_package_register_powercap()
1341 rp->power_zone = power_zone; in rapl_package_register_powercap()
1348 return -ENODEV; in rapl_package_register_powercap()
1351 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { in rapl_package_register_powercap()
1352 struct powercap_zone *parent = rp->power_zone; in rapl_package_register_powercap()
1354 if (rd->id == RAPL_DOMAIN_PACKAGE) in rapl_package_register_powercap()
1356 if (rd->id == RAPL_DOMAIN_PLATFORM) in rapl_package_register_powercap()
1360 power_zone = powercap_register_zone(&rd->power_zone, in rapl_package_register_powercap()
1361 rp->priv->control_type, in rapl_package_register_powercap()
1362 rd->name, parent, in rapl_package_register_powercap()
1363 &zone_ops[rd->id], nr_pl, in rapl_package_register_powercap()
1368 rp->name, rd->name); in rapl_package_register_powercap()
1380 while (--rd >= rp->domains) { in rapl_package_register_powercap()
1381 pr_debug("unregister %s domain %s\n", rp->name, rd->name); in rapl_package_register_powercap()
1382 powercap_unregister_zone(rp->priv->control_type, in rapl_package_register_powercap()
1383 &rd->power_zone); in rapl_package_register_powercap()
1389 static int rapl_check_domain(int domain, struct rapl_package *rp) in rapl_check_domain() argument
1399 ra.reg = rp->priv->regs[domain][RAPL_DOMAIN_REG_STATUS]; in rapl_check_domain()
1403 return -EINVAL; in rapl_check_domain()
1405 /* make sure domain counters are available and contains non-zero in rapl_check_domain()
1410 if (rp->priv->read_raw(get_rid(rp), &ra) || !ra.value) in rapl_check_domain()
1411 return -ENODEV; in rapl_check_domain()
1423 struct rapl_defaults *defaults = get_defaults(rd->rp); in rapl_get_domain_unit()
1426 if (!rd->regs[RAPL_DOMAIN_REG_UNIT].val) { in rapl_get_domain_unit()
1427 if (!rd->rp->priv->reg_unit.val) { in rapl_get_domain_unit()
1429 return -ENODEV; in rapl_get_domain_unit()
1431 rd->regs[RAPL_DOMAIN_REG_UNIT] = rd->rp->priv->reg_unit; in rapl_get_domain_unit()
1434 if (!defaults->check_unit) { in rapl_get_domain_unit()
1436 return -ENODEV; in rapl_get_domain_unit()
1439 ret = defaults->check_unit(rd); in rapl_get_domain_unit()
1443 if (rd->id == RAPL_DOMAIN_DRAM && defaults->dram_domain_energy_unit) in rapl_get_domain_unit()
1444 rd->energy_unit = defaults->dram_domain_energy_unit; in rapl_get_domain_unit()
1445 if (rd->id == RAPL_DOMAIN_PLATFORM && defaults->psys_domain_energy_unit) in rapl_get_domain_unit()
1446 rd->energy_unit = defaults->psys_domain_energy_unit; in rapl_get_domain_unit()
1452 * 1. Locked by BIOS, in this case we still provide read-only access so that
1467 rd->rpl[i].locked = true; in rapl_detect_powerlimit()
1469 rd->rp->name, rd->name, pl_names[i]); in rapl_detect_powerlimit()
1474 rd->rpl[i].name = NULL; in rapl_detect_powerlimit()
1481 static int rapl_detect_domains(struct rapl_package *rp) in rapl_detect_domains() argument
1488 if (!rapl_check_domain(i, rp)) { in rapl_detect_domains()
1489 rp->domain_map |= 1 << i; in rapl_detect_domains()
1493 rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX); in rapl_detect_domains()
1494 if (!rp->nr_domains) { in rapl_detect_domains()
1495 pr_debug("no valid rapl domains found in %s\n", rp->name); in rapl_detect_domains()
1496 return -ENODEV; in rapl_detect_domains()
1498 pr_debug("found %d domains on %s\n", rp->nr_domains, rp->name); in rapl_detect_domains()
1500 rp->domains = kcalloc(rp->nr_domains, sizeof(struct rapl_domain), in rapl_detect_domains()
1502 if (!rp->domains) in rapl_detect_domains()
1503 return -ENOMEM; in rapl_detect_domains()
1505 rapl_init_domains(rp); in rapl_detect_domains()
1507 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { in rapl_detect_domains()
1538 * 2.3283064365386962890625e-10
1540 * energy units. Use 2.3283064365386962890625e-10 (2^-32) Joules as
1570 static int get_pmu_cpu(struct rapl_package *rp) in get_pmu_cpu() argument
1574 if (!rp->has_pmu) in get_pmu_cpu()
1578 if (rp->priv->type != RAPL_IF_TPMI) in get_pmu_cpu()
1583 if (topology_physical_package_id(cpu) == rp->id) in get_pmu_cpu()
1589 static bool is_rp_pmu_cpu(struct rapl_package *rp, int cpu) in is_rp_pmu_cpu() argument
1591 if (!rp->has_pmu) in is_rp_pmu_cpu()
1595 if (rp->priv->type != RAPL_IF_TPMI) in is_rp_pmu_cpu()
1599 return topology_physical_package_id(cpu) == rp->id; in is_rp_pmu_cpu()
1604 struct rapl_package *rp = event->pmu_private; in event_to_pmu_data() local
1606 return &rp->pmu_data; in event_to_pmu_data()
1613 struct rapl_package *rp = event->pmu_private; in event_read_counter() local
1618 if (event->hw.idx < 0) in event_read_counter()
1621 ret = rapl_read_data_raw(&rp->domains[event->hw.idx], ENERGY_COUNTER, false, &val); in event_read_counter()
1634 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) in __rapl_pmu_event_start()
1637 event->hw.state = 0; in __rapl_pmu_event_start()
1639 list_add_tail(&event->active_entry, &data->active_list); in __rapl_pmu_event_start()
1641 local64_set(&event->hw.prev_count, event_read_counter(event)); in __rapl_pmu_event_start()
1642 if (++data->n_active == 1) in __rapl_pmu_event_start()
1643 hrtimer_start(&data->hrtimer, data->timer_interval, in __rapl_pmu_event_start()
1652 raw_spin_lock_irqsave(&data->lock, flags); in rapl_pmu_event_start()
1654 raw_spin_unlock_irqrestore(&data->lock, flags); in rapl_pmu_event_start()
1659 struct hw_perf_event *hwc = &event->hw; in rapl_event_update()
1665 * Follow the generic code to drain hwc->prev_count. in rapl_event_update()
1668 prev_raw_count = local64_read(&hwc->prev_count); in rapl_event_update()
1671 } while (!local64_try_cmpxchg(&hwc->prev_count, in rapl_event_update()
1678 * (event-)time and add that to the generic event. in rapl_event_update()
1680 delta = new_raw_count - prev_raw_count; in rapl_event_update()
1683 * Scale delta to smallest unit (2^-32) in rapl_event_update()
1685 * or use ldexp(count, -32). in rapl_event_update()
1688 sdelta = delta * data->scale[event->hw.flags]; in rapl_event_update()
1690 local64_add(sdelta, &event->count); in rapl_event_update()
1698 struct hw_perf_event *hwc = &event->hw; in rapl_pmu_event_stop()
1701 raw_spin_lock_irqsave(&data->lock, flags); in rapl_pmu_event_stop()
1704 if (!(hwc->state & PERF_HES_STOPPED)) { in rapl_pmu_event_stop()
1705 WARN_ON_ONCE(data->n_active <= 0); in rapl_pmu_event_stop()
1706 if (--data->n_active == 0) in rapl_pmu_event_stop()
1707 hrtimer_cancel(&data->hrtimer); in rapl_pmu_event_stop()
1709 list_del(&event->active_entry); in rapl_pmu_event_stop()
1711 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); in rapl_pmu_event_stop()
1712 hwc->state |= PERF_HES_STOPPED; in rapl_pmu_event_stop()
1716 if ((mode & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { in rapl_pmu_event_stop()
1722 hwc->state |= PERF_HES_UPTODATE; in rapl_pmu_event_stop()
1725 raw_spin_unlock_irqrestore(&data->lock, flags); in rapl_pmu_event_stop()
1731 struct hw_perf_event *hwc = &event->hw; in rapl_pmu_event_add()
1734 raw_spin_lock_irqsave(&data->lock, flags); in rapl_pmu_event_add()
1736 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; in rapl_pmu_event_add()
1741 raw_spin_unlock_irqrestore(&data->lock, flags); in rapl_pmu_event_add()
1772 struct rapl_package *pos, *rp = NULL; in rapl_pmu_event_init() local
1773 u64 cfg = event->attr.config & RAPL_EVENT_MASK; in rapl_pmu_event_init()
1777 if (event->attr.type != event->pmu->type) in rapl_pmu_event_init()
1778 return -ENOENT; in rapl_pmu_event_init()
1782 return -EINVAL; in rapl_pmu_event_init()
1784 if (event->cpu < 0) in rapl_pmu_event_init()
1785 return -EINVAL; in rapl_pmu_event_init()
1789 if (is_rp_pmu_cpu(pos, event->cpu)) { in rapl_pmu_event_init()
1790 rp = pos; in rapl_pmu_event_init()
1794 if (!rp) in rapl_pmu_event_init()
1795 return -ENODEV; in rapl_pmu_event_init()
1800 event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG; in rapl_pmu_event_init()
1801 event->pmu_private = rp; /* Which package */ in rapl_pmu_event_init()
1802 event->hw.flags = domain; /* Which domain */ in rapl_pmu_event_init()
1804 event->hw.idx = -1; in rapl_pmu_event_init()
1805 /* Find out the index in rp->domains[] to get domain pointer */ in rapl_pmu_event_init()
1806 for (idx = 0; idx < rp->nr_domains; idx++) { in rapl_pmu_event_init()
1807 if (rp->domains[idx].id == domain) { in rapl_pmu_event_init()
1808 event->hw.idx = idx; in rapl_pmu_event_init()
1828 if (!data->n_active) in rapl_hrtimer_handle()
1831 raw_spin_lock_irqsave(&data->lock, flags); in rapl_hrtimer_handle()
1833 list_for_each_entry(event, &data->active_list, active_entry) in rapl_hrtimer_handle()
1836 raw_spin_unlock_irqrestore(&data->lock, flags); in rapl_hrtimer_handle()
1838 hrtimer_forward_now(hrtimer, data->timer_interval); in rapl_hrtimer_handle()
1861 struct rapl_package *rp; in cpumask_show() local
1867 return -ENOMEM; in cpumask_show()
1874 list_for_each_entry(rp, &rapl_packages, plist) { in cpumask_show()
1875 cpu = get_pmu_cpu(rp); in cpumask_show()
1899 PMU_FORMAT_ATTR(event, "config:0-7");
1923 RAPL_EVENT_ATTR_STR(energy-cores, rapl_cores, "event=0x01");
1924 RAPL_EVENT_ATTR_STR(energy-pkg, rapl_pkg, "event=0x02");
1925 RAPL_EVENT_ATTR_STR(energy-ram, rapl_ram, "event=0x03");
1926 RAPL_EVENT_ATTR_STR(energy-gpu, rapl_gpu, "event=0x04");
1927 RAPL_EVENT_ATTR_STR(energy-psys, rapl_psys, "event=0x05");
1929 RAPL_EVENT_ATTR_STR(energy-cores.unit, rapl_unit_cores, "Joules");
1930 RAPL_EVENT_ATTR_STR(energy-pkg.unit, rapl_unit_pkg, "Joules");
1931 RAPL_EVENT_ATTR_STR(energy-ram.unit, rapl_unit_ram, "Joules");
1932 RAPL_EVENT_ATTR_STR(energy-gpu.unit, rapl_unit_gpu, "Joules");
1933 RAPL_EVENT_ATTR_STR(energy-psys.unit, rapl_unit_psys, "Joules");
1935 RAPL_EVENT_ATTR_STR(energy-cores.scale, rapl_scale_cores, "2.3283064365386962890625e-10");
1936 RAPL_EVENT_ATTR_STR(energy-pkg.scale, rapl_scale_pkg, "2.3283064365386962890625e-10");
1937 RAPL_EVENT_ATTR_STR(energy-ram.scale, rapl_scale_ram, "2.3283064365386962890625e-10");
1938 RAPL_EVENT_ATTR_STR(energy-gpu.scale, rapl_scale_gpu, "2.3283064365386962890625e-10");
1939 RAPL_EVENT_ATTR_STR(energy-psys.scale, rapl_scale_psys, "2.3283064365386962890625e-10");
1950 return rapl_pmu.domain_map & BIT(domain) ? attr->mode : 0; \
1973 static int rapl_pmu_update(struct rapl_package *rp) in rapl_pmu_update() argument
1978 if (rapl_pmu.registered && !(rp->domain_map & (~rapl_pmu.domain_map))) in rapl_pmu_update()
1986 rapl_pmu.domain_map |= rp->domain_map; in rapl_pmu_update()
2000 ret = perf_pmu_register(&rapl_pmu.pmu, "power", -1); in rapl_pmu_update()
2008 rp->has_pmu = true; in rapl_pmu_update()
2012 int rapl_package_add_pmu(struct rapl_package *rp) in rapl_package_add_pmu() argument
2014 struct rapl_package_pmu_data *data = &rp->pmu_data; in rapl_package_add_pmu()
2017 if (rp->has_pmu) in rapl_package_add_pmu()
2018 return -EEXIST; in rapl_package_add_pmu()
2022 for (idx = 0; idx < rp->nr_domains; idx++) { in rapl_package_add_pmu()
2023 struct rapl_domain *rd = &rp->domains[idx]; in rapl_package_add_pmu()
2024 int domain = rd->id; in rapl_package_add_pmu()
2027 if (!test_bit(domain, &rp->domain_map)) in rapl_package_add_pmu()
2031 * The RAPL PMU granularity is 2^-32 Joules in rapl_package_add_pmu()
2032 * data->scale[]: times of 2^-32 Joules for each ENERGY COUNTER increase in rapl_package_add_pmu()
2034 val = rd->energy_unit * (1ULL << 32); in rapl_package_add_pmu()
2036 data->scale[domain] = val; in rapl_package_add_pmu()
2039 struct rapl_primitive_info *rpi = get_rpi(rp, ENERGY_COUNTER); in rapl_package_add_pmu()
2046 * max_count = rpi->mask >> rpi->shift + 1 in rapl_package_add_pmu()
2047 * max_energy_pj = max_count * rd->energy_unit in rapl_package_add_pmu()
2052 val = (rpi->mask >> rpi->shift) + 1; in rapl_package_add_pmu()
2053 val *= rd->energy_unit; in rapl_package_add_pmu()
2060 pr_debug("Domain %s: hw unit %lld * 2^-32 Joules\n", rd->name, data->scale[domain]); in rapl_package_add_pmu()
2064 raw_spin_lock_init(&data->lock); in rapl_package_add_pmu()
2065 INIT_LIST_HEAD(&data->active_list); in rapl_package_add_pmu()
2066 data->timer_interval = ms_to_ktime(rapl_pmu.timer_ms); in rapl_package_add_pmu()
2067 hrtimer_init(&data->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in rapl_package_add_pmu()
2068 data->hrtimer.function = rapl_hrtimer_handle; in rapl_package_add_pmu()
2070 return rapl_pmu_update(rp); in rapl_package_add_pmu()
2074 void rapl_package_remove_pmu(struct rapl_package *rp) in rapl_package_remove_pmu() argument
2078 if (!rp->has_pmu) in rapl_package_remove_pmu()
2085 if (pos->has_pmu && pos != rp) in rapl_package_remove_pmu()
2096 void rapl_remove_package_cpuslocked(struct rapl_package *rp) in rapl_remove_package_cpuslocked() argument
2100 package_power_limit_irq_restore(rp); in rapl_remove_package_cpuslocked()
2102 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { in rapl_remove_package_cpuslocked()
2110 if (rd->id == RAPL_DOMAIN_PACKAGE) { in rapl_remove_package_cpuslocked()
2115 rp->name, rd->name); in rapl_remove_package_cpuslocked()
2116 powercap_unregister_zone(rp->priv->control_type, in rapl_remove_package_cpuslocked()
2117 &rd->power_zone); in rapl_remove_package_cpuslocked()
2120 powercap_unregister_zone(rp->priv->control_type, in rapl_remove_package_cpuslocked()
2121 &rd_package->power_zone); in rapl_remove_package_cpuslocked()
2122 list_del(&rp->plist); in rapl_remove_package_cpuslocked()
2123 kfree(rp); in rapl_remove_package_cpuslocked()
2127 void rapl_remove_package(struct rapl_package *rp) in rapl_remove_package() argument
2130 rapl_remove_package_cpuslocked(rp); in rapl_remove_package()
2136 * 1. AMD/HYGON platforms use per-PKG package energy counter
2138 * 2.1 CLX-AP platform has per-DIE package energy counter
2140 * package energy counter can be considered as per-PKG/per-DIE,
2141 * here it is considered as per-DIE.
2153 struct rapl_package *rp; in rapl_find_package_domain_cpuslocked() local
2167 list_for_each_entry(rp, &rapl_packages, plist) { in rapl_find_package_domain_cpuslocked()
2168 if (rp->id == uid in rapl_find_package_domain_cpuslocked()
2169 && rp->priv->control_type == priv->control_type) in rapl_find_package_domain_cpuslocked()
2170 return rp; in rapl_find_package_domain_cpuslocked()
2187 struct rapl_package *rp; in rapl_add_package_cpuslocked() local
2190 rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL); in rapl_add_package_cpuslocked()
2191 if (!rp) in rapl_add_package_cpuslocked()
2192 return ERR_PTR(-ENOMEM); in rapl_add_package_cpuslocked()
2195 rp->id = rapl_msrs_are_pkg_scope() ? in rapl_add_package_cpuslocked()
2197 if ((int)(rp->id) < 0) { in rapl_add_package_cpuslocked()
2199 return ERR_PTR(-EINVAL); in rapl_add_package_cpuslocked()
2201 rp->lead_cpu = id; in rapl_add_package_cpuslocked()
2203 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d-die-%d", in rapl_add_package_cpuslocked()
2206 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", in rapl_add_package_cpuslocked()
2209 rp->id = id; in rapl_add_package_cpuslocked()
2210 rp->lead_cpu = -1; in rapl_add_package_cpuslocked()
2211 snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d", id); in rapl_add_package_cpuslocked()
2214 rp->priv = priv; in rapl_add_package_cpuslocked()
2215 ret = rapl_config(rp); in rapl_add_package_cpuslocked()
2220 if (rapl_detect_domains(rp)) { in rapl_add_package_cpuslocked()
2221 ret = -ENODEV; in rapl_add_package_cpuslocked()
2224 ret = rapl_package_register_powercap(rp); in rapl_add_package_cpuslocked()
2226 INIT_LIST_HEAD(&rp->plist); in rapl_add_package_cpuslocked()
2227 list_add(&rp->plist, &rapl_packages); in rapl_add_package_cpuslocked()
2228 return rp; in rapl_add_package_cpuslocked()
2232 kfree(rp->domains); in rapl_add_package_cpuslocked()
2233 kfree(rp); in rapl_add_package_cpuslocked()
2247 struct rapl_package *rp; in power_limit_state_save() local
2252 list_for_each_entry(rp, &rapl_packages, plist) { in power_limit_state_save()
2253 if (!rp->power_zone) in power_limit_state_save()
2255 rd = power_zone_to_rapl_domain(rp->power_zone); in power_limit_state_save()
2258 &rd->rpl[i].last_power_limit); in power_limit_state_save()
2260 rd->rpl[i].last_power_limit = 0; in power_limit_state_save()
2268 struct rapl_package *rp; in power_limit_state_restore() local
2273 list_for_each_entry(rp, &rapl_packages, plist) { in power_limit_state_restore()
2274 if (!rp->power_zone) in power_limit_state_restore()
2276 rd = power_zone_to_rapl_domain(rp->power_zone); in power_limit_state_restore()
2278 if (rd->rpl[i].last_power_limit) in power_limit_state_restore()
2280 rd->rpl[i].last_power_limit); in power_limit_state_restore()
2312 defaults_msr = (struct rapl_defaults *)id->driver_data; in rapl_init()
2316 return -ENOMEM; in rapl_init()