1 // SPDX-License-Identifier: GPL-2.0
3  * Copyright (c) 2017-2019, The Linux Foundation. All rights reserved.
16 #include <linux/nvmem-consumer.h>
21 #include <linux/soc/qcom/llcc-qcom.h>
71  * struct llcc_slice_config - Data associated with the llcc slice
73  * @slice_id: llcc slice id for each client
87  *                    When configured to 0 all ways in llcc are probed.
97  * @stale_cap_en: Bit enables stale only if current scid is over-cap.
98  * @mru_uncap_en: Roll-over on reserved cache ways if current scid is
99  *                under-cap.
100  * @mru_rollover: Roll-over on reserved cache ways.
101  * @alloc_oneway_en: Allways allocate one way on over-cap even if there's no
102  *                   same-scid lines for replacement.
103  * @ovcap_en: Once current scid is over-capacity, allocate other over-cap SCID.
104  * @ovcap_prio: Once current scid is over-capacity, allocate other low priority
105  *              over-cap scid. Depends on corresponding bit being set in
107  * @vict_prio: When current scid is under-capacity, allocate over other
108  *             lower-than victim priority-line threshold scid.
3120 	/* LLCC Common registers */
3125 	/* LLCC DRP registers */
3147 	/* LLCC Common registers */
3152 	/* LLCC DRP registers */
3164 /* LLCC register offset starting from v1.0.0 */
3170 /* LLCC register offset starting from v2.0.1 */
3497 static struct llcc_drv_data *drv_data = (void *) -EPROBE_DEFER;
3500  * llcc_slice_getd - get llcc slice descriptor
3503  * A pointer to llcc slice descriptor will be returned on success
3515 	cfg = drv_data->cfg;  in llcc_slice_getd()
3516 	sz = drv_data->cfg_size;  in llcc_slice_getd()
3519 		if (cfg->usecase_id == uid)  in llcc_slice_getd()
3523 		return ERR_PTR(-ENODEV);  in llcc_slice_getd()
3527 		return ERR_PTR(-ENOMEM);  in llcc_slice_getd()
3529 	desc->slice_id = cfg->slice_id;  in llcc_slice_getd()
3530 	desc->slice_size = cfg->max_cap;  in llcc_slice_getd()
3537  * llcc_slice_putd - llcc slice descriptor
3538  * @desc: Pointer to llcc slice descriptor
3566 	ret = regmap_write(drv_data->bcast_regmap, act_ctrl_reg,  in llcc_update_act_ctrl()
3573 	ret = regmap_write(drv_data->bcast_regmap, act_ctrl_reg,  in llcc_update_act_ctrl()
3578 	if (drv_data->version >= LLCC_VERSION_4_1_0_0) {  in llcc_update_act_ctrl()
3579 		regmap = drv_data->bcast_and_regmap ?: drv_data->bcast_regmap;  in llcc_update_act_ctrl()
3587 	ret = regmap_read_poll_timeout(drv_data->bcast_regmap, status_reg,  in llcc_update_act_ctrl()
3593 	if (drv_data->version >= LLCC_VERSION_4_1_0_0)  in llcc_update_act_ctrl()
3594 		ret = regmap_write(drv_data->bcast_regmap, act_clear_reg,  in llcc_update_act_ctrl()
3601  * llcc_slice_activate - Activate the llcc slice
3602  * @desc: Pointer to llcc slice descriptor
3616 		return -EINVAL;  in llcc_slice_activate()
3618 	mutex_lock(&drv_data->lock);  in llcc_slice_activate()
3619 	if (test_bit(desc->slice_id, drv_data->bitmap)) {  in llcc_slice_activate()
3620 		mutex_unlock(&drv_data->lock);  in llcc_slice_activate()
3626 	ret = llcc_update_act_ctrl(desc->slice_id, act_ctrl_val,  in llcc_slice_activate()
3629 		mutex_unlock(&drv_data->lock);  in llcc_slice_activate()
3633 	__set_bit(desc->slice_id, drv_data->bitmap);  in llcc_slice_activate()
3634 	mutex_unlock(&drv_data->lock);  in llcc_slice_activate()
3641  * llcc_slice_deactivate - Deactivate the llcc slice
3642  * @desc: Pointer to llcc slice descriptor
3656 		return -EINVAL;  in llcc_slice_deactivate()
3658 	mutex_lock(&drv_data->lock);  in llcc_slice_deactivate()
3659 	if (!test_bit(desc->slice_id, drv_data->bitmap)) {  in llcc_slice_deactivate()
3660 		mutex_unlock(&drv_data->lock);  in llcc_slice_deactivate()
3665 	ret = llcc_update_act_ctrl(desc->slice_id, act_ctrl_val,  in llcc_slice_deactivate()
3668 		mutex_unlock(&drv_data->lock);  in llcc_slice_deactivate()
3672 	__clear_bit(desc->slice_id, drv_data->bitmap);  in llcc_slice_deactivate()
3673 	mutex_unlock(&drv_data->lock);  in llcc_slice_deactivate()
3680  * llcc_get_slice_id - return the slice id
3681  * @desc: Pointer to llcc slice descriptor
3686 		return -EINVAL;  in llcc_get_slice_id()
3688 	return desc->slice_id;  in llcc_get_slice_id()
3693  * llcc_get_slice_size - return the slice id
3694  * @desc: Pointer to llcc slice descriptor
3701 	return desc->slice_size;  in llcc_get_slice_size()
3718 	attr1_val = config->cache_mode;  in _qcom_llcc_cfg_program()
3719 	attr1_val |= config->probe_target_ways << ATTR1_PROBE_TARGET_WAYS_SHIFT;  in _qcom_llcc_cfg_program()
3720 	attr1_val |= config->fixed_size << ATTR1_FIXED_SIZE_SHIFT;  in _qcom_llcc_cfg_program()
3721 	attr1_val |= config->priority << ATTR1_PRIORITY_SHIFT;  in _qcom_llcc_cfg_program()
3723 	max_cap_cacheline = MAX_CAP_TO_BYTES(config->max_cap);  in _qcom_llcc_cfg_program()
3726 	 * LLCC instances can vary for each target.  in _qcom_llcc_cfg_program()
3728 	 * to each llcc instance (llcc0,.. llccN).  in _qcom_llcc_cfg_program()
3730 	 * llcc instances, we need to configure the max cap accordingly.  in _qcom_llcc_cfg_program()
3732 	max_cap_cacheline = max_cap_cacheline / drv_data->num_banks;  in _qcom_llcc_cfg_program()
3734 	if (cfg->max_cap_shift)  in _qcom_llcc_cfg_program()
3735 		attr1_val |= max_cap_cacheline << cfg->max_cap_shift;  in _qcom_llcc_cfg_program()
3739 	attr1_cfg = LLCC_TRP_ATTR1_CFGn(config->slice_id);  in _qcom_llcc_cfg_program()
3741 	ret = regmap_write(drv_data->bcast_regmap, attr1_cfg, attr1_val);  in _qcom_llcc_cfg_program()
3745 	if (drv_data->version >= LLCC_VERSION_4_1_0_0) {  in _qcom_llcc_cfg_program()
3746 		attr2_cfg = LLCC_TRP_ATTR2_CFGn(config->slice_id);  in _qcom_llcc_cfg_program()
3747 		attr0_val = config->res_ways;  in _qcom_llcc_cfg_program()
3748 		attr2_val = config->bonus_ways;  in _qcom_llcc_cfg_program()
3750 		attr0_val = config->res_ways & ATTR0_RES_WAYS_MASK;  in _qcom_llcc_cfg_program()
3751 		attr0_val |= config->bonus_ways << ATTR0_BONUS_WAYS_SHIFT;  in _qcom_llcc_cfg_program()
3754 	attr0_cfg = LLCC_TRP_ATTR0_CFGn(config->slice_id);  in _qcom_llcc_cfg_program()
3756 	ret = regmap_write(drv_data->bcast_regmap, attr0_cfg, attr0_val);  in _qcom_llcc_cfg_program()
3760 	if (drv_data->version >= LLCC_VERSION_4_1_0_0) {  in _qcom_llcc_cfg_program()
3761 		ret = regmap_write(drv_data->bcast_regmap, attr2_cfg, attr2_val);  in _qcom_llcc_cfg_program()
3766 	/* At least SDM845 disallows non-secure writes to these registers */  in _qcom_llcc_cfg_program()
3767 	if (!cfg->skip_llcc_cfg) {  in _qcom_llcc_cfg_program()
3770 		disable_cap_alloc = config->dis_cap_alloc << config->slice_id;  in _qcom_llcc_cfg_program()
3771 		ret = regmap_update_bits(drv_data->bcast_regmap, LLCC_TRP_SCID_DIS_CAP_ALLOC,  in _qcom_llcc_cfg_program()
3772 					 BIT(config->slice_id), disable_cap_alloc);  in _qcom_llcc_cfg_program()
3776 		if (drv_data->version < LLCC_VERSION_4_1_0_0) {  in _qcom_llcc_cfg_program()
3777 			retain_pc = config->retain_on_pc << config->slice_id;  in _qcom_llcc_cfg_program()
3778 			ret = regmap_update_bits(drv_data->bcast_regmap, LLCC_TRP_PCB_ACT,  in _qcom_llcc_cfg_program()
3779 						 BIT(config->slice_id), retain_pc);  in _qcom_llcc_cfg_program()
3785 	if (drv_data->version >= LLCC_VERSION_2_0_0_0) {  in _qcom_llcc_cfg_program()
3788 		wren = config->write_scid_en << config->slice_id;  in _qcom_llcc_cfg_program()
3789 		ret = regmap_update_bits(drv_data->bcast_regmap, LLCC_TRP_WRSC_EN,  in _qcom_llcc_cfg_program()
3790 					 BIT(config->slice_id), wren);  in _qcom_llcc_cfg_program()
3795 	if (drv_data->version >= LLCC_VERSION_2_1_0_0) {  in _qcom_llcc_cfg_program()
3798 		wr_cache_en = config->write_scid_cacheable_en << config->slice_id;  in _qcom_llcc_cfg_program()
3799 		ret = regmap_update_bits(drv_data->bcast_regmap, LLCC_TRP_WRSC_CACHEABLE_EN,  in _qcom_llcc_cfg_program()
3800 					 BIT(config->slice_id), wr_cache_en);  in _qcom_llcc_cfg_program()
3805 	if (drv_data->version >= LLCC_VERSION_4_1_0_0) {  in _qcom_llcc_cfg_program()
3815 		stale_en = config->stale_en << config->slice_id;  in _qcom_llcc_cfg_program()
3816 		ret = regmap_update_bits(drv_data->bcast_regmap, LLCC_TRP_ALGO_CFG1,  in _qcom_llcc_cfg_program()
3817 					 BIT(config->slice_id), stale_en);  in _qcom_llcc_cfg_program()
3821 		stale_cap_en = config->stale_cap_en << config->slice_id;  in _qcom_llcc_cfg_program()
3822 		ret = regmap_update_bits(drv_data->bcast_regmap, LLCC_TRP_ALGO_CFG2,  in _qcom_llcc_cfg_program()
3823 					 BIT(config->slice_id), stale_cap_en);  in _qcom_llcc_cfg_program()
3827 		mru_uncap_en = config->mru_uncap_en << config->slice_id;  in _qcom_llcc_cfg_program()
3828 		ret = regmap_update_bits(drv_data->bcast_regmap, LLCC_TRP_ALGO_CFG3,  in _qcom_llcc_cfg_program()
3829 					 BIT(config->slice_id), mru_uncap_en);  in _qcom_llcc_cfg_program()
3833 		mru_rollover = config->mru_rollover << config->slice_id;  in _qcom_llcc_cfg_program()
3834 		ret = regmap_update_bits(drv_data->bcast_regmap, LLCC_TRP_ALGO_CFG4,  in _qcom_llcc_cfg_program()
3835 					 BIT(config->slice_id), mru_rollover);  in _qcom_llcc_cfg_program()
3839 		alloc_oneway_en = config->alloc_oneway_en << config->slice_id;  in _qcom_llcc_cfg_program()
3840 		ret = regmap_update_bits(drv_data->bcast_regmap, LLCC_TRP_ALGO_CFG5,  in _qcom_llcc_cfg_program()
3841 					 BIT(config->slice_id), alloc_oneway_en);  in _qcom_llcc_cfg_program()
3845 		ovcap_en = config->ovcap_en << config->slice_id;  in _qcom_llcc_cfg_program()
3846 		ret = regmap_update_bits(drv_data->bcast_regmap, LLCC_TRP_ALGO_CFG6,  in _qcom_llcc_cfg_program()
3847 					 BIT(config->slice_id), ovcap_en);  in _qcom_llcc_cfg_program()
3851 		ovcap_prio = config->ovcap_prio << config->slice_id;  in _qcom_llcc_cfg_program()
3852 		ret = regmap_update_bits(drv_data->bcast_regmap, LLCC_TRP_ALGO_CFG7,  in _qcom_llcc_cfg_program()
3853 					 BIT(config->slice_id), ovcap_prio);  in _qcom_llcc_cfg_program()
3857 		vict_prio = config->vict_prio << config->slice_id;  in _qcom_llcc_cfg_program()
3858 		ret = regmap_update_bits(drv_data->bcast_regmap, LLCC_TRP_ALGO_CFG8,  in _qcom_llcc_cfg_program()
3859 					 BIT(config->slice_id), vict_prio);  in _qcom_llcc_cfg_program()
3864 	if (config->activate_on_init) {  in _qcom_llcc_cfg_program()
3865 		desc.slice_id = config->slice_id;  in _qcom_llcc_cfg_program()
3880 	sz = drv_data->cfg_size;  in qcom_llcc_cfg_program()
3881 	llcc_table = drv_data->cfg;  in qcom_llcc_cfg_program()
3896 	ret = nvmem_cell_read_u8(&pdev->dev, "multi-chan-ddr", cfg_index);  in qcom_llcc_get_cfg_index()
3897 	if (ret == -ENOENT || ret == -EOPNOTSUPP) {  in qcom_llcc_get_cfg_index()
3899 			return -EINVAL;  in qcom_llcc_get_cfg_index()
3905 		ret = -EINVAL;  in qcom_llcc_get_cfg_index()
3913 	drv_data = ERR_PTR(-ENODEV);  in qcom_llcc_remove()
3932 	return devm_regmap_init_mmio(&pdev->dev, base, &llcc_regmap_config);  in qcom_llcc_init_mmio()
3938 	struct device *dev = &pdev->dev;  in qcom_llcc_probe()
3950 		return -EBUSY;  in qcom_llcc_probe()
3954 		ret = -ENOMEM;  in qcom_llcc_probe()
3958 	/* Initialize the first LLCC bank regmap */  in qcom_llcc_probe()
3965 	cfgs = of_device_get_match_data(&pdev->dev);  in qcom_llcc_probe()
3967 		ret = -EINVAL;  in qcom_llcc_probe()
3970 	ret = qcom_llcc_get_cfg_index(pdev, &cfg_index, cfgs->num_config);  in qcom_llcc_probe()
3973 	cfg = &cfgs->llcc_config[cfg_index];  in qcom_llcc_probe()
3975 	if (cfg->num_banks) {  in qcom_llcc_probe()
3976 		num_banks = cfg->num_banks;  in qcom_llcc_probe()
3978 		ret = regmap_read(regmap, cfg->reg_offset[LLCC_COMMON_STATUS0], &num_banks);  in qcom_llcc_probe()
3986 	drv_data->num_banks = num_banks;  in qcom_llcc_probe()
3988 	drv_data->regmaps = devm_kcalloc(dev, num_banks, sizeof(*drv_data->regmaps), GFP_KERNEL);  in qcom_llcc_probe()
3989 	if (!drv_data->regmaps) {  in qcom_llcc_probe()
3990 		ret = -ENOMEM;  in qcom_llcc_probe()
3994 	drv_data->regmaps[0] = regmap;  in qcom_llcc_probe()
3996 	/* Initialize rest of LLCC bank regmaps */  in qcom_llcc_probe()
3998 		char *base __free(kfree) = kasprintf(GFP_KERNEL, "llcc%d_base", i);  in qcom_llcc_probe()
4000 		drv_data->regmaps[i] = qcom_llcc_init_mmio(pdev, i, base);  in qcom_llcc_probe()
4001 		if (IS_ERR(drv_data->regmaps[i])) {  in qcom_llcc_probe()
4002 			ret = PTR_ERR(drv_data->regmaps[i]);  in qcom_llcc_probe()
4007 	drv_data->bcast_regmap = qcom_llcc_init_mmio(pdev, i, "llcc_broadcast_base");  in qcom_llcc_probe()
4008 	if (IS_ERR(drv_data->bcast_regmap)) {  in qcom_llcc_probe()
4009 		if (cfg->no_broadcast_register) {  in qcom_llcc_probe()
4010 			drv_data->bcast_regmap = regmap;  in qcom_llcc_probe()
4012 			ret = PTR_ERR(drv_data->bcast_regmap);  in qcom_llcc_probe()
4018 	ret = regmap_read(drv_data->bcast_regmap, cfg->reg_offset[LLCC_COMMON_HW_INFO],  in qcom_llcc_probe()
4023 	drv_data->version = version;  in qcom_llcc_probe()
4025 	/* Applicable only when drv_data->version >= 4.1 */  in qcom_llcc_probe()
4026 	if (drv_data->version >= LLCC_VERSION_4_1_0_0) {  in qcom_llcc_probe()
4027 		drv_data->bcast_and_regmap = qcom_llcc_init_mmio(pdev, i + 1, "llcc_broadcast_and_base");  in qcom_llcc_probe()
4028 		if (IS_ERR(drv_data->bcast_and_regmap)) {  in qcom_llcc_probe()
4029 			ret = PTR_ERR(drv_data->bcast_and_regmap);  in qcom_llcc_probe()
4030 			if (ret == -EINVAL)  in qcom_llcc_probe()
4031 				drv_data->bcast_and_regmap = NULL;  in qcom_llcc_probe()
4037 	llcc_cfg = cfg->sct_data;  in qcom_llcc_probe()
4038 	sz = cfg->size;  in qcom_llcc_probe()
4041 		if (llcc_cfg[i].slice_id > drv_data->max_slices)  in qcom_llcc_probe()
4042 			drv_data->max_slices = llcc_cfg[i].slice_id;  in qcom_llcc_probe()
4044 	drv_data->bitmap = devm_bitmap_zalloc(dev, drv_data->max_slices,  in qcom_llcc_probe()
4046 	if (!drv_data->bitmap) {  in qcom_llcc_probe()
4047 		ret = -ENOMEM;  in qcom_llcc_probe()
4051 	drv_data->cfg = llcc_cfg;  in qcom_llcc_probe()
4052 	drv_data->cfg_size = sz;  in qcom_llcc_probe()
4053 	drv_data->edac_reg_offset = cfg->edac_reg_offset;  in qcom_llcc_probe()
4054 	drv_data->ecc_irq_configured = cfg->irq_configured;  in qcom_llcc_probe()
4055 	mutex_init(&drv_data->lock);  in qcom_llcc_probe()
4062 	drv_data->ecc_irq = platform_get_irq_optional(pdev, 0);  in qcom_llcc_probe()
4070 	if (!cfg->no_edac) {  in qcom_llcc_probe()
4071 		llcc_edac = platform_device_register_data(&pdev->dev,  in qcom_llcc_probe()
4072 						"qcom_llcc_edac", -1, drv_data,  in qcom_llcc_probe()
4075 			dev_err(dev, "Failed to register llcc edac driver\n");  in qcom_llcc_probe()
4080 	drv_data = ERR_PTR(-ENODEV);  in qcom_llcc_probe()
4085 	{ .compatible = "qcom,ipq5424-llcc", .data = &ipq5424_cfgs},
4086 	{ .compatible = "qcom,qcs615-llcc", .data = &qcs615_cfgs},
4087 	{ .compatible = "qcom,qcs8300-llcc", .data = &qcs8300_cfgs},
4088 	{ .compatible = "qcom,qdu1000-llcc", .data = &qdu1000_cfgs},
4089 	{ .compatible = "qcom,sa8775p-llcc", .data = &sa8775p_cfgs },
4090 	{ .compatible = "qcom,sar1130p-llcc", .data = &sar1130p_cfgs },
4091 	{ .compatible = "qcom,sar2130p-llcc", .data = &sar2130p_cfgs },
4092 	{ .compatible = "qcom,sc7180-llcc", .data = &sc7180_cfgs },
4093 	{ .compatible = "qcom,sc7280-llcc", .data = &sc7280_cfgs },
4094 	{ .compatible = "qcom,sc8180x-llcc", .data = &sc8180x_cfgs },
4095 	{ .compatible = "qcom,sc8280xp-llcc", .data = &sc8280xp_cfgs },
4096 	{ .compatible = "qcom,sdm845-llcc", .data = &sdm845_cfgs },
4097 	{ .compatible = "qcom,sm6350-llcc", .data = &sm6350_cfgs },
4098 	{ .compatible = "qcom,sm7150-llcc", .data = &sm7150_cfgs },
4099 	{ .compatible = "qcom,sm8150-llcc", .data = &sm8150_cfgs },
4100 	{ .compatible = "qcom,sm8250-llcc", .data = &sm8250_cfgs },
4101 	{ .compatible = "qcom,sm8350-llcc", .data = &sm8350_cfgs },
4102 	{ .compatible = "qcom,sm8450-llcc", .data = &sm8450_cfgs },
4103 	{ .compatible = "qcom,sm8550-llcc", .data = &sm8550_cfgs },
4104 	{ .compatible = "qcom,sm8650-llcc", .data = &sm8650_cfgs },
4105 	{ .compatible = "qcom,x1e80100-llcc", .data = &x1e80100_cfgs },
4112 		.name = "qcom-llcc",