1 // SPDX-License-Identifier: GPL-2.0
315 /* define the SEC's dfx regs region and region length */
334 	struct hisi_qm *qm = s->private;  in sec_diff_regs_show()
336 	hisi_qm_acc_diff_regs_dump(qm, s, qm->debug.acc_diff_regs,  in sec_diff_regs_show()
358 MODULE_PARM_DESC(pf_q_num, "Number of queues in PF(v1 2-4096, v2 2-1024)");
366 		return -EINVAL;  in sec_ctx_q_num_set()
370 		return -EINVAL;  in sec_ctx_q_num_set()
374 		return -EINVAL;  in sec_ctx_q_num_set()
395 MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");
426 	cap_val_h = qm->cap_tables.dev_cap_table[high].cap_val;  in sec_get_alg_bitmap()
427 	cap_val_l = qm->cap_tables.dev_cap_table[low].cap_val;  in sec_get_alg_bitmap()
456 	reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG);  in sec_set_endian()
464 	writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG);  in sec_set_endian()
471 	if (qm->ver > QM_HW_V2) {  in sec_engine_sva_config()
472 		reg = readl_relaxed(qm->io_base +  in sec_engine_sva_config()
475 		writel_relaxed(reg, qm->io_base +  in sec_engine_sva_config()
478 		reg = readl_relaxed(qm->io_base +  in sec_engine_sva_config()
482 		writel_relaxed(reg, qm->io_base +  in sec_engine_sva_config()
485 		reg = readl_relaxed(qm->io_base +  in sec_engine_sva_config()
488 		writel_relaxed(reg, qm->io_base +  in sec_engine_sva_config()
490 		reg = readl_relaxed(qm->io_base +  in sec_engine_sva_config()
493 		if (qm->use_sva)  in sec_engine_sva_config()
497 		writel_relaxed(reg, qm->io_base +  in sec_engine_sva_config()
507 	if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))  in sec_open_sva_prefetch()
511 	val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG);  in sec_open_sva_prefetch()
513 	writel(val, qm->io_base + SEC_PREFETCH_CFG);  in sec_open_sva_prefetch()
515 	ret = readl_relaxed_poll_timeout(qm->io_base + SEC_PREFETCH_CFG,  in sec_open_sva_prefetch()
519 		pci_err(qm->pdev, "failed to open sva prefetch\n");  in sec_open_sva_prefetch()
527 	if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))  in sec_close_sva_prefetch()
530 	val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG);  in sec_close_sva_prefetch()
532 	writel(val, qm->io_base + SEC_PREFETCH_CFG);  in sec_close_sva_prefetch()
534 	ret = readl_relaxed_poll_timeout(qm->io_base + SEC_SVA_TRANS,  in sec_close_sva_prefetch()
538 		pci_err(qm->pdev, "failed to close sva prefetch\n");  in sec_close_sva_prefetch()
545 	if (qm->ver < QM_HW_V3)  in sec_enable_clock_gate()
548 	val = readl_relaxed(qm->io_base + SEC_CONTROL_REG);  in sec_enable_clock_gate()
550 	writel_relaxed(val, qm->io_base + SEC_CONTROL_REG);  in sec_enable_clock_gate()
552 	val = readl(qm->io_base + SEC_DYNAMIC_GATE_REG);  in sec_enable_clock_gate()
554 	writel(val, qm->io_base + SEC_DYNAMIC_GATE_REG);  in sec_enable_clock_gate()
556 	val = readl(qm->io_base + SEC_CORE_AUTO_GATE);  in sec_enable_clock_gate()
558 	writel(val, qm->io_base + SEC_CORE_AUTO_GATE);  in sec_enable_clock_gate()
566 	val = readl_relaxed(qm->io_base + SEC_CONTROL_REG);  in sec_disable_clock_gate()
568 	writel_relaxed(val, qm->io_base + SEC_CONTROL_REG);  in sec_disable_clock_gate()
579 	writel_relaxed(0x1, qm->io_base + SEC_MEM_START_INIT_REG);  in sec_engine_init()
581 	ret = readl_relaxed_poll_timeout(qm->io_base + SEC_MEM_INIT_DONE_REG,  in sec_engine_init()
585 		pci_err(qm->pdev, "fail to init sec mem\n");  in sec_engine_init()
589 	reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG);  in sec_engine_init()
591 	writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG);  in sec_engine_init()
596 	       qm->io_base + AM_CFG_SINGLE_PORT_MAX_TRANS);  in sec_engine_init()
598 	reg = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CORE_ENABLE_BITMAP, qm->cap_ver);  in sec_engine_init()
599 	writel(reg, qm->io_base + SEC_SAA_EN_REG);  in sec_engine_init()
601 	if (qm->ver < QM_HW_V3) {  in sec_engine_init()
604 			       qm->io_base + SEC_BD_ERR_CHK_EN_REG0);  in sec_engine_init()
608 			       qm->io_base + SEC_BD_ERR_CHK_EN_REG1);  in sec_engine_init()
610 			       qm->io_base + SEC_BD_ERR_CHK_EN_REG3);  in sec_engine_init()
624 	writel(AXUSER_BASE, qm->io_base + QM_ARUSER_M_CFG_1);  in sec_set_user_domain_and_cache()
625 	writel(ARUSER_M_CFG_ENABLE, qm->io_base + QM_ARUSER_M_CFG_ENABLE);  in sec_set_user_domain_and_cache()
626 	writel(AXUSER_BASE, qm->io_base + QM_AWUSER_M_CFG_1);  in sec_set_user_domain_and_cache()
627 	writel(AWUSER_M_CFG_ENABLE, qm->io_base + QM_AWUSER_M_CFG_ENABLE);  in sec_set_user_domain_and_cache()
628 	writel(WUSER_M_CFG_ENABLE, qm->io_base + QM_WUSER_M_CFG_ENABLE);  in sec_set_user_domain_and_cache()
631 	writel(AXI_M_CFG, qm->io_base + QM_AXI_M_CFG);  in sec_set_user_domain_and_cache()
632 	writel(AXI_M_CFG_ENABLE, qm->io_base + QM_AXI_M_CFG_ENABLE);  in sec_set_user_domain_and_cache()
634 	/* disable FLR triggered by BME(bus master enable) */  in sec_set_user_domain_and_cache()
635 	writel(PEH_AXUSER_CFG, qm->io_base + QM_PEH_AXUSER_CFG);  in sec_set_user_domain_and_cache()
636 	writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE);  in sec_set_user_domain_and_cache()
641 	       FIELD_PREP(CQC_CACHE_WB_THRD, 1), qm->io_base + QM_CACHE_CTL);  in sec_set_user_domain_and_cache()
646 /* sec_debug_regs_clear() - clear the sec debug regs */
651 	/* clear sec dfx regs */  in sec_debug_regs_clear()
652 	writel(0x1, qm->io_base + SEC_CTRL_CNT_CLR_CE);  in sec_debug_regs_clear()
654 		readl(qm->io_base + sec_dfx_regs[i].offset);  in sec_debug_regs_clear()
657 	writel(0x0, qm->io_base + SEC_CTRL_CNT_CLR_CE);  in sec_debug_regs_clear()
666 	val1 = readl(qm->io_base + SEC_CONTROL_REG);  in sec_master_ooo_ctrl()
670 					   SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);  in sec_master_ooo_ctrl()
676 	if (qm->ver > QM_HW_V2)  in sec_master_ooo_ctrl()
677 		writel(val2, qm->io_base + SEC_OOO_SHUTDOWN_SEL);  in sec_master_ooo_ctrl()
679 	writel(val1, qm->io_base + SEC_CONTROL_REG);  in sec_master_ooo_ctrl()
686 	if (qm->ver == QM_HW_V1) {  in sec_hw_error_enable()
687 		writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK);  in sec_hw_error_enable()
688 		pci_info(qm->pdev, "V1 not support hw error handle\n");  in sec_hw_error_enable()
692 	ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CE_MASK_CAP, qm->cap_ver);  in sec_hw_error_enable()
693 	nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver);  in sec_hw_error_enable()
696 	writel(ce | nfe | SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_CORE_INT_SOURCE);  in sec_hw_error_enable()
699 	writel(ce, qm->io_base + SEC_RAS_CE_REG);  in sec_hw_error_enable()
700 	writel(SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_RAS_FE_REG);  in sec_hw_error_enable()
701 	writel(nfe, qm->io_base + SEC_RAS_NFE_REG);  in sec_hw_error_enable()
707 	writel(ce | nfe | SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_CORE_INT_MASK);  in sec_hw_error_enable()
713 	writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK);  in sec_hw_error_disable()
719 	writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_CE_REG);  in sec_hw_error_disable()
720 	writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_FE_REG);  in sec_hw_error_disable()
721 	writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_NFE_REG);  in sec_hw_error_disable()
726 	return readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) &  in sec_clear_enable_read()
735 		return -EINVAL;  in sec_clear_enable_write()
737 	tmp = (readl(qm->io_base + SEC_CTRL_CNT_CLR_CE) &  in sec_clear_enable_write()
739 	writel(tmp, qm->io_base + SEC_CTRL_CNT_CLR_CE);  in sec_clear_enable_write()
747 	struct sec_debug_file *file = filp->private_data;  in sec_debug_read()
749 	struct hisi_qm *qm = file->qm;  in sec_debug_read()
757 	spin_lock_irq(&file->lock);  in sec_debug_read()
759 	switch (file->index) {  in sec_debug_read()
767 	spin_unlock_irq(&file->lock);  in sec_debug_read()
774 	spin_unlock_irq(&file->lock);  in sec_debug_read()
776 	return -EINVAL;  in sec_debug_read()
782 	struct sec_debug_file *file = filp->private_data;  in sec_debug_write()
784 	struct hisi_qm *qm = file->qm;  in sec_debug_write()
792 		return -ENOSPC;  in sec_debug_write()
794 	len = simple_write_to_buffer(tbuf, SEC_DBGFS_VAL_MAX_LEN - 1,  in sec_debug_write()
801 		return -EFAULT;  in sec_debug_write()
807 	spin_lock_irq(&file->lock);  in sec_debug_write()
809 	switch (file->index) {  in sec_debug_write()
816 		ret = -EINVAL;  in sec_debug_write()
823 	spin_unlock_irq(&file->lock);  in sec_debug_write()
845 		return -EINVAL;  in sec_debugfs_atomic64_set()
857 	hisi_qm_regs_dump(s, s->private);  in sec_regs_show()
866 	struct hisi_qm *qm = s->private;  in sec_cap_regs_show()
869 	size = qm->cap_tables.qm_cap_size;  in sec_cap_regs_show()
871 		seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.qm_cap_table[i].name,  in sec_cap_regs_show()
872 			   qm->cap_tables.qm_cap_table[i].cap_val);  in sec_cap_regs_show()
874 	size = qm->cap_tables.dev_cap_size;  in sec_cap_regs_show()
876 		seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.dev_cap_table[i].name,  in sec_cap_regs_show()
877 			   qm->cap_tables.dev_cap_table[i].cap_val);  in sec_cap_regs_show()
886 	struct dfx_diff_registers *sec_regs = qm->debug.acc_diff_regs;  in sec_core_debug_init()
888 	struct device *dev = &qm->pdev->dev;  in sec_core_debug_init()
889 	struct sec_dfx *dfx = &sec->debug.dfx;  in sec_core_debug_init()  local
894 	tmp_d = debugfs_create_dir("sec_dfx", qm->debug.debug_root);  in sec_core_debug_init()
898 		return -ENOMEM;  in sec_core_debug_init()
900 	regset->regs = sec_dfx_regs;  in sec_core_debug_init()
901 	regset->nregs = ARRAY_SIZE(sec_dfx_regs);  in sec_core_debug_init()
902 	regset->base = qm->io_base;  in sec_core_debug_init()
903 	regset->dev = dev;  in sec_core_debug_init()
905 	if (qm->pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF)  in sec_core_debug_init()
907 	if (qm->fun_type == QM_HW_PF && sec_regs)  in sec_core_debug_init()
912 		atomic64_t *data = (atomic64_t *)((uintptr_t)dfx +  in sec_core_debug_init()
919 			    qm->debug.debug_root, qm, &sec_cap_regs_fops);  in sec_core_debug_init()
929 	if (qm->pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF) {  in sec_debug_init()
931 			spin_lock_init(&sec->debug.files[i].lock);  in sec_debug_init()
932 			sec->debug.files[i].index = i;  in sec_debug_init()
933 			sec->debug.files[i].qm = qm;  in sec_debug_init()
936 						  qm->debug.debug_root,  in sec_debug_init()
937 						  sec->debug.files + i,  in sec_debug_init()
947 	struct device *dev = &qm->pdev->dev;  in sec_debugfs_init()
956 	qm->debug.debug_root = debugfs_create_dir(dev_name(dev),  in sec_debugfs_init()
958 	qm->debug.sqe_mask_offset = SEC_SQE_MASK_OFFSET;  in sec_debugfs_init()
959 	qm->debug.sqe_mask_len = SEC_SQE_MASK_LEN;  in sec_debugfs_init()
970 	debugfs_remove_recursive(qm->debug.debug_root);  in sec_debugfs_init()
977 	debugfs_remove_recursive(qm->debug.debug_root);  in sec_debugfs_exit()
984 	struct qm_debug *debug = &qm->debug;  in sec_show_last_regs_init()
987 	debug->last_words = kcalloc(ARRAY_SIZE(sec_dfx_regs),  in sec_show_last_regs_init()
989 	if (!debug->last_words)  in sec_show_last_regs_init()
990 		return -ENOMEM;  in sec_show_last_regs_init()
993 		debug->last_words[i] = readl_relaxed(qm->io_base +  in sec_show_last_regs_init()
1001 	struct qm_debug *debug = &qm->debug;  in sec_show_last_regs_uninit()
1003 	if (qm->fun_type == QM_HW_VF || !debug->last_words)  in sec_show_last_regs_uninit()
1006 	kfree(debug->last_words);  in sec_show_last_regs_uninit()
1007 	debug->last_words = NULL;  in sec_show_last_regs_uninit()
1012 	struct qm_debug *debug = &qm->debug;  in sec_show_last_dfx_regs()
1013 	struct pci_dev *pdev = qm->pdev;  in sec_show_last_dfx_regs()
1017 	if (qm->fun_type == QM_HW_VF || !debug->last_words)  in sec_show_last_dfx_regs()
1022 		val = readl_relaxed(qm->io_base + sec_dfx_regs[i].offset);  in sec_show_last_dfx_regs()
1023 		if (val != debug->last_words[i])  in sec_show_last_dfx_regs()
1025 				sec_dfx_regs[i].name, debug->last_words[i], val);  in sec_show_last_dfx_regs()
1032 	struct device *dev = &qm->pdev->dev;  in sec_log_hw_error()
1035 	while (errs->msg) {  in sec_log_hw_error()
1036 		if (errs->int_msk & err_sts) {  in sec_log_hw_error()
1038 					errs->msg, errs->int_msk);  in sec_log_hw_error()
1040 			if (SEC_CORE_INT_STATUS_M_ECC & errs->int_msk) {  in sec_log_hw_error()
1041 				err_val = readl(qm->io_base +  in sec_log_hw_error()
1054 	return readl(qm->io_base + SEC_CORE_INT_STATUS);  in sec_get_hw_err_status()
1059 	writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE);  in sec_clear_hw_err_status()
1066 	nfe_mask = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver);  in sec_disable_error_report()
1067 	writel(nfe_mask & (~err_type), qm->io_base + SEC_RAS_NFE_REG);  in sec_disable_error_report()
1074 	val = readl(qm->io_base + SEC_CONTROL_REG);  in sec_open_axi_master_ooo()
1075 	writel(val & SEC_AXI_SHUTDOWN_DISABLE, qm->io_base + SEC_CONTROL_REG);  in sec_open_axi_master_ooo()
1076 	writel(val | SEC_AXI_SHUTDOWN_ENABLE, qm->io_base + SEC_CONTROL_REG);  in sec_open_axi_master_ooo()
1085 		if (err_status & qm->err_info.ecc_2bits_mask)  in sec_get_err_result()
1086 			qm->err_status.is_dev_ecc_mbit = true;  in sec_get_err_result()
1089 		if (err_status & qm->err_info.dev_reset_mask) {  in sec_get_err_result()
1105 	if (err_status & qm->err_info.dev_shutdown_mask)  in sec_dev_is_abnormal()
1113 	struct hisi_qm_err_info *err_info = &qm->err_info;  in sec_err_info_init()
1115 	err_info->fe = SEC_RAS_FE_ENB_MSK;  in sec_err_info_init()
1116 	err_info->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_CE_MASK_CAP, qm->cap_ver);  in sec_err_info_init()
1117 	err_info->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_NFE_MASK_CAP, qm->cap_ver);  in sec_err_info_init()
1118 	err_info->ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC;  in sec_err_info_init()
1119 	err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info,  in sec_err_info_init()
1120 				     SEC_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);  in sec_err_info_init()
1121 	err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info,  in sec_err_info_init()
1122 			SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);  in sec_err_info_init()
1123 	err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info,  in sec_err_info_init()
1124 			SEC_QM_RESET_MASK_CAP, qm->cap_ver);  in sec_err_info_init()
1125 	err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info,  in sec_err_info_init()
1126 			SEC_RESET_MASK_CAP, qm->cap_ver);  in sec_err_info_init()
1127 	err_info->msi_wr_port = BIT(0);  in sec_err_info_init()
1128 	err_info->acpi_rst = "SRST";  in sec_err_info_init()
1148 	struct hisi_qm *qm = &sec->qm;  in sec_pf_probe_init()
1160 		pci_err(qm->pdev, "Failed to init last word regs!\n");  in sec_pf_probe_init()
1168 	struct pci_dev *pdev = qm->pdev;  in sec_pre_store_cap_reg()
1172 	sec_cap = devm_kzalloc(&pdev->dev, sizeof(*sec_cap) * size, GFP_KERNEL);  in sec_pre_store_cap_reg()
1174 		return -ENOMEM;  in sec_pre_store_cap_reg()
1180 				     i, qm->cap_ver);  in sec_pre_store_cap_reg()
1183 	qm->cap_tables.dev_cap_table = sec_cap;  in sec_pre_store_cap_reg()
1184 	qm->cap_tables.dev_cap_size = size;  in sec_pre_store_cap_reg()
1194 	qm->pdev = pdev;  in sec_qm_init()
1195 	qm->mode = uacce_mode;  in sec_qm_init()
1196 	qm->sqe_size = SEC_SQE_SIZE;  in sec_qm_init()
1197 	qm->dev_name = sec_name;  in sec_qm_init()
1199 	qm->fun_type = (pdev->device == PCI_DEVICE_ID_HUAWEI_SEC_PF) ?  in sec_qm_init()
1201 	if (qm->fun_type == QM_HW_PF) {  in sec_qm_init()
1202 		qm->qp_base = SEC_PF_DEF_Q_BASE;  in sec_qm_init()
1203 		qm->qp_num = pf_q_num;  in sec_qm_init()
1204 		qm->debug.curr_qm_qp_num = pf_q_num;  in sec_qm_init()
1205 		qm->qm_list = &sec_devices;  in sec_qm_init()
1206 		qm->err_ini = &sec_err_ini;  in sec_qm_init()
1208 			set_bit(QM_MODULE_PARAM, &qm->misc_ctl);  in sec_qm_init()
1209 	} else if (qm->fun_type == QM_HW_VF && qm->ver == QM_HW_V1) {  in sec_qm_init()
1216 		qm->qp_base = SEC_PF_DEF_Q_NUM;  in sec_qm_init()
1217 		qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM;  in sec_qm_init()
1222 		pci_err(qm->pdev, "Failed to init sec qm configures!\n");  in sec_qm_init()
1229 		pci_err(qm->pdev, "Failed to pre-store capability registers!\n");  in sec_qm_init()
1236 		pci_err(qm->pdev, "Failed to set sec algs!\n");  in sec_qm_init()
1251 	struct hisi_qm *qm = &sec->qm;  in sec_probe_init()
1254 	if (qm->fun_type == QM_HW_PF) {  in sec_probe_init()
1259 		if (qm->ver >= QM_HW_V3) {  in sec_probe_init()
1261 			qm->type_rate = type_rate;  in sec_probe_init()
1270 	if (qm->fun_type == QM_HW_VF)  in sec_probe_uninit()
1282 	struct device *dev = &sec->qm.pdev->dev;  in sec_iommu_used_check()
1287 	sec->iommu_used = false;  in sec_iommu_used_check()
1289 		if (domain->type & __IOMMU_DOMAIN_PAGING)  in sec_iommu_used_check()
1290 			sec->iommu_used = true;  in sec_iommu_used_check()
1292 			domain->type);  in sec_iommu_used_check()
1302 	sec = devm_kzalloc(&pdev->dev, sizeof(*sec), GFP_KERNEL);  in sec_probe()
1304 		return -ENOMEM;  in sec_probe()
1306 	qm = &sec->qm;  in sec_probe()
1313 	sec->ctx_q_num = ctx_q_num;  in sec_probe()
1339 	if (qm->uacce) {  in sec_probe()
1340 		ret = uacce_register(qm->uacce);  in sec_probe()
1347 	if (qm->fun_type == QM_HW_PF && vfs_num) {  in sec_probe()
1379 	if (qm->fun_type == QM_HW_PF && qm->vfs_num)  in sec_remove()