1 // SPDX-License-Identifier: GPL-2.0-or-later
5  * Copyright (C) 2017-2023 Broadcom Inc.
6  *  (mailto: mpi3mr-[email protected])
11 #include <linux/io-64-nonatomic-lo-hi.h>
23 MODULE_PARM_DESC(poll_queues, "Number of queues for io_uring poll mode. (Range 1 - 126)");
46 	pi = op_req_q->pi;  in mpi3mr_check_req_qfull()
47 	ci = READ_ONCE(op_req_q->ci);  in mpi3mr_check_req_qfull()
48 	max_entries = op_req_q->num_requests;  in mpi3mr_check_req_qfull()
50 	if ((ci == (pi + 1)) || ((!ci) && (pi == (max_entries - 1))))  in mpi3mr_check_req_qfull()
60 	max_vectors = mrioc->intr_info_count;  in mpi3mr_sync_irqs()
63 		synchronize_irq(pci_irq_vector(mrioc->pdev, i));  in mpi3mr_sync_irqs()
68 	mrioc->intr_enabled = 0;  in mpi3mr_ioc_disable_intr()
74 	mrioc->intr_enabled = 1;  in mpi3mr_ioc_enable_intr()
83 	if (!mrioc->intr_info)  in mpi3mr_cleanup_isr()
86 	for (i = 0; i < mrioc->intr_info_count; i++)  in mpi3mr_cleanup_isr()
87 		free_irq(pci_irq_vector(mrioc->pdev, i),  in mpi3mr_cleanup_isr()
88 		    (mrioc->intr_info + i));  in mpi3mr_cleanup_isr()
90 	kfree(mrioc->intr_info);  in mpi3mr_cleanup_isr()
91 	mrioc->intr_info = NULL;  in mpi3mr_cleanup_isr()
92 	mrioc->intr_info_count = 0;  in mpi3mr_cleanup_isr()
93 	mrioc->is_intr_info_set = false;  in mpi3mr_cleanup_isr()
94 	pci_free_irq_vectors(mrioc->pdev);  in mpi3mr_cleanup_isr()
102 	sgel->flags = flags;  in mpi3mr_add_sg_single()
103 	sgel->length = cpu_to_le32(length);  in mpi3mr_add_sg_single()
104 	sgel->address = cpu_to_le64(dma_addr);  in mpi3mr_add_sg_single()
111 	mpi3mr_add_sg_single(paddr, sgl_flags, 0, -1);  in mpi3mr_build_zero_len_sge()
120 	if ((phys_addr < mrioc->reply_buf_dma) ||  in mpi3mr_get_reply_virt_addr()
121 	    (phys_addr > mrioc->reply_buf_dma_max_address))  in mpi3mr_get_reply_virt_addr()
124 	return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);  in mpi3mr_get_reply_virt_addr()
133 	return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);  in mpi3mr_get_sensebuf_virt_addr()
142 	spin_lock_irqsave(&mrioc->reply_free_queue_lock, flags);  in mpi3mr_repost_reply_buf()
143 	old_idx  =  mrioc->reply_free_queue_host_index;  in mpi3mr_repost_reply_buf()
144 	mrioc->reply_free_queue_host_index = (  in mpi3mr_repost_reply_buf()
145 	    (mrioc->reply_free_queue_host_index ==  in mpi3mr_repost_reply_buf()
146 	    (mrioc->reply_free_qsz - 1)) ? 0 :  in mpi3mr_repost_reply_buf()
147 	    (mrioc->reply_free_queue_host_index + 1));  in mpi3mr_repost_reply_buf()
148 	mrioc->reply_free_q[old_idx] = cpu_to_le64(reply_dma);  in mpi3mr_repost_reply_buf()
149 	writel(mrioc->reply_free_queue_host_index,  in mpi3mr_repost_reply_buf()
150 	    &mrioc->sysif_regs->reply_free_host_index);  in mpi3mr_repost_reply_buf()
151 	spin_unlock_irqrestore(&mrioc->reply_free_queue_lock, flags);  in mpi3mr_repost_reply_buf()
160 	spin_lock_irqsave(&mrioc->sbq_lock, flags);  in mpi3mr_repost_sense_buf()
161 	old_idx  =  mrioc->sbq_host_index;  in mpi3mr_repost_sense_buf()
162 	mrioc->sbq_host_index = ((mrioc->sbq_host_index ==  in mpi3mr_repost_sense_buf()
163 	    (mrioc->sense_buf_q_sz - 1)) ? 0 :  in mpi3mr_repost_sense_buf()
164 	    (mrioc->sbq_host_index + 1));  in mpi3mr_repost_sense_buf()
165 	mrioc->sense_buf_q[old_idx] = cpu_to_le64(sense_buf_dma);  in mpi3mr_repost_sense_buf()
166 	writel(mrioc->sbq_host_index,  in mpi3mr_repost_sense_buf()
167 	    &mrioc->sysif_regs->sense_buffer_free_host_index);  in mpi3mr_repost_sense_buf()
168 	spin_unlock_irqrestore(&mrioc->sbq_lock, flags);  in mpi3mr_repost_sense_buf()
177 	event = event_reply->event;  in mpi3mr_print_event_data()
198 		    (struct mpi3_device_page0 *)event_reply->event_data;  in mpi3mr_print_event_data()
200 		    event_data->dev_handle, event_data->device_form);  in mpi3mr_print_event_data()
206 		    (struct mpi3_device_page0 *)event_reply->event_data;  in mpi3mr_print_event_data()
208 		    event_data->dev_handle, event_data->device_form);  in mpi3mr_print_event_data()
214 		    (struct mpi3_event_data_device_status_change *)event_reply->event_data;  in mpi3mr_print_event_data()
216 		    event_data->dev_handle, event_data->reason_code);  in mpi3mr_print_event_data()
222 		    (struct mpi3_event_data_sas_discovery *)event_reply->event_data;  in mpi3mr_print_event_data()
224 		    (event_data->reason_code == MPI3_EVENT_SAS_DISC_RC_STARTED) ?  in mpi3mr_print_event_data()
226 		    le32_to_cpu(event_data->discovery_status));  in mpi3mr_print_event_data()
265 		    (struct mpi3_event_data_pcie_enumeration *)event_reply->event_data;  in mpi3mr_print_event_data()
267 		    (event_data->reason_code ==  in mpi3mr_print_event_data()
269 		if (event_data->enumeration_status)  in mpi3mr_print_event_data()
271 			    le32_to_cpu(event_data->enumeration_status));  in mpi3mr_print_event_data()
294 	mrioc->change_count = le16_to_cpu(event_reply->ioc_change_count);  in mpi3mr_handle_events()
307 		return &mrioc->init_cmds;  in mpi3mr_get_drv_cmd()
309 		return &mrioc->cfg_cmds;  in mpi3mr_get_drv_cmd()
311 		return &mrioc->bsg_cmds;  in mpi3mr_get_drv_cmd()
313 		return &mrioc->host_tm_cmds;  in mpi3mr_get_drv_cmd()
315 		return &mrioc->pel_abort_cmd;  in mpi3mr_get_drv_cmd()
317 		return &mrioc->pel_cmds;  in mpi3mr_get_drv_cmd()
319 		return &mrioc->transport_cmds;  in mpi3mr_get_drv_cmd()
321 		if (def_reply && def_reply->function ==  in mpi3mr_get_drv_cmd()
330 		idx = host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;  in mpi3mr_get_drv_cmd()
331 		return &mrioc->dev_rmhs_cmds[idx];  in mpi3mr_get_drv_cmd()
336 		idx = host_tag - MPI3MR_HOSTTAG_EVTACKCMD_MIN;  in mpi3mr_get_drv_cmd()
337 		return &mrioc->evtack_cmds[idx];  in mpi3mr_get_drv_cmd()
360 	reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_admin_reply_desc()
365 		host_tag = le16_to_cpu(status_desc->host_tag);  in mpi3mr_process_admin_reply_desc()
366 		ioc_status = le16_to_cpu(status_desc->ioc_status);  in mpi3mr_process_admin_reply_desc()
369 			ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);  in mpi3mr_process_admin_reply_desc()
375 		*reply_dma = le64_to_cpu(addr_desc->reply_frame_address);  in mpi3mr_process_admin_reply_desc()
379 		host_tag = le16_to_cpu(def_reply->host_tag);  in mpi3mr_process_admin_reply_desc()
380 		ioc_status = le16_to_cpu(def_reply->ioc_status);  in mpi3mr_process_admin_reply_desc()
383 			ioc_loginfo = le32_to_cpu(def_reply->ioc_log_info);  in mpi3mr_process_admin_reply_desc()
385 		if (def_reply->function == MPI3_FUNCTION_SCSI_IO) {  in mpi3mr_process_admin_reply_desc()
388 			    le64_to_cpu(scsi_reply->sense_data_buffer_address));  in mpi3mr_process_admin_reply_desc()
389 			sense_count = le32_to_cpu(scsi_reply->sense_count);  in mpi3mr_process_admin_reply_desc()
401 		host_tag = le16_to_cpu(success_desc->host_tag);  in mpi3mr_process_admin_reply_desc()
409 		if (cmdptr->state & MPI3MR_CMD_PENDING) {  in mpi3mr_process_admin_reply_desc()
410 			cmdptr->state |= MPI3MR_CMD_COMPLETE;  in mpi3mr_process_admin_reply_desc()
411 			cmdptr->ioc_loginfo = ioc_loginfo;  in mpi3mr_process_admin_reply_desc()
413 				cmdptr->ioc_status = ioc_status;  in mpi3mr_process_admin_reply_desc()
415 				cmdptr->ioc_status = masked_ioc_status;  in mpi3mr_process_admin_reply_desc()
416 			cmdptr->state &= ~MPI3MR_CMD_PENDING;  in mpi3mr_process_admin_reply_desc()
418 				cmdptr->state |= MPI3MR_CMD_REPLY_VALID;  in mpi3mr_process_admin_reply_desc()
419 				memcpy((u8 *)cmdptr->reply, (u8 *)def_reply,  in mpi3mr_process_admin_reply_desc()
420 				    mrioc->reply_sz);  in mpi3mr_process_admin_reply_desc()
422 			if (sense_buf && cmdptr->sensebuf) {  in mpi3mr_process_admin_reply_desc()
423 				cmdptr->is_sense = 1;  in mpi3mr_process_admin_reply_desc()
424 				memcpy(cmdptr->sensebuf, sense_buf,  in mpi3mr_process_admin_reply_desc()
427 			if (cmdptr->is_waiting) {  in mpi3mr_process_admin_reply_desc()
428 				complete(&cmdptr->done);  in mpi3mr_process_admin_reply_desc()
429 				cmdptr->is_waiting = 0;  in mpi3mr_process_admin_reply_desc()
430 			} else if (cmdptr->callback)  in mpi3mr_process_admin_reply_desc()
431 				cmdptr->callback(mrioc, cmdptr);  in mpi3mr_process_admin_reply_desc()
437 		    le64_to_cpu(scsi_reply->sense_data_buffer_address));  in mpi3mr_process_admin_reply_desc()
442 	u32 exp_phase = mrioc->admin_reply_ephase;  in mpi3mr_process_admin_reply_q()
443 	u32 admin_reply_ci = mrioc->admin_reply_ci;  in mpi3mr_process_admin_reply_q()
449 	if (!atomic_add_unless(&mrioc->admin_reply_q_in_use, 1, 1))  in mpi3mr_process_admin_reply_q()
452 	reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +  in mpi3mr_process_admin_reply_q()
455 	if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_admin_reply_q()
457 		atomic_dec(&mrioc->admin_reply_q_in_use);  in mpi3mr_process_admin_reply_q()
462 		if (mrioc->unrecoverable || mrioc->io_admin_reset_sync)  in mpi3mr_process_admin_reply_q()
465 		mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci);  in mpi3mr_process_admin_reply_q()
471 		if (++admin_reply_ci == mrioc->num_admin_replies) {  in mpi3mr_process_admin_reply_q()
476 		    (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +  in mpi3mr_process_admin_reply_q()
478 		if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_admin_reply_q()
483 			    &mrioc->sysif_regs->admin_reply_queue_ci);  in mpi3mr_process_admin_reply_q()
488 	writel(admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);  in mpi3mr_process_admin_reply_q()
489 	mrioc->admin_reply_ci = admin_reply_ci;  in mpi3mr_process_admin_reply_q()
490 	mrioc->admin_reply_ephase = exp_phase;  in mpi3mr_process_admin_reply_q()
491 	atomic_dec(&mrioc->admin_reply_q_in_use);  in mpi3mr_process_admin_reply_q()
497  * mpi3mr_get_reply_desc - get reply descriptor frame corresponding to
498  *	queue's consumer index from operational reply descriptor queue.
500  * @reply_ci: operational reply descriptor's queue consumer index
508 	struct segments *segments = op_reply_q->q_segments;  in mpi3mr_get_reply_desc()
512 	    segments[reply_ci / op_reply_q->segment_qd].segment;  in mpi3mr_get_reply_desc()
514 	    (reply_ci % op_reply_q->segment_qd);  in mpi3mr_get_reply_desc()
519  * mpi3mr_process_op_reply_q - Operational reply queue handler
521  * @op_reply_q: Operational reply queue info
523  * Checks the specific operational reply queue and drains the
541 	reply_qidx = op_reply_q->qid - 1;  in mpi3mr_process_op_reply_q()
543 	if (!atomic_add_unless(&op_reply_q->in_use, 1, 1))  in mpi3mr_process_op_reply_q()
546 	exp_phase = op_reply_q->ephase;  in mpi3mr_process_op_reply_q()
547 	reply_ci = op_reply_q->ci;  in mpi3mr_process_op_reply_q()
550 	if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_op_reply_q()
552 		atomic_dec(&op_reply_q->in_use);  in mpi3mr_process_op_reply_q()
557 		if (mrioc->unrecoverable || mrioc->io_admin_reset_sync)  in mpi3mr_process_op_reply_q()
560 		req_q_idx = le16_to_cpu(reply_desc->request_queue_id) - 1;  in mpi3mr_process_op_reply_q()
561 		op_req_q = &mrioc->req_qinfo[req_q_idx];  in mpi3mr_process_op_reply_q()
563 		WRITE_ONCE(op_req_q->ci, le16_to_cpu(reply_desc->request_queue_ci));  in mpi3mr_process_op_reply_q()
566 		atomic_dec(&op_reply_q->pend_ios);  in mpi3mr_process_op_reply_q()
572 		if (++reply_ci == op_reply_q->num_replies) {  in mpi3mr_process_op_reply_q()
579 		if ((le16_to_cpu(reply_desc->reply_flags) &  in mpi3mr_process_op_reply_q()
587 		if (num_op_reply > mrioc->max_host_ios) {  in mpi3mr_process_op_reply_q()
588 			op_reply_q->enable_irq_poll = true;  in mpi3mr_process_op_reply_q()
594 			    &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);  in mpi3mr_process_op_reply_q()
595 			atomic_sub(threshold_comps, &op_reply_q->pend_ios);  in mpi3mr_process_op_reply_q()
601 	    &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);  in mpi3mr_process_op_reply_q()
602 	op_reply_q->ci = reply_ci;  in mpi3mr_process_op_reply_q()
603 	op_reply_q->ephase = exp_phase;  in mpi3mr_process_op_reply_q()
604 	atomic_sub(threshold_comps, &op_reply_q->pend_ios);  in mpi3mr_process_op_reply_q()
605 	atomic_dec(&op_reply_q->in_use);  in mpi3mr_process_op_reply_q()
610  * mpi3mr_blk_mq_poll - Operational reply queue handler
614  * Checks the specific operational reply queue and drains the
626 	mrioc = (struct mpi3mr_ioc *)shost->hostdata;  in mpi3mr_blk_mq_poll()
628 	if ((mrioc->reset_in_progress || mrioc->prepare_for_reset ||  in mpi3mr_blk_mq_poll()
629 	    mrioc->unrecoverable || mrioc->pci_err_recovery))  in mpi3mr_blk_mq_poll()
633 			&mrioc->op_reply_qinfo[queue_num]);  in mpi3mr_blk_mq_poll()
648 	mrioc = intr_info->mrioc;  in mpi3mr_isr_primary()
650 	if (!mrioc->intr_enabled)  in mpi3mr_isr_primary()
653 	midx = intr_info->msix_index;  in mpi3mr_isr_primary()
657 	if (intr_info->op_reply_q)  in mpi3mr_isr_primary()
659 		    intr_info->op_reply_q);  in mpi3mr_isr_primary()
684 	if (!intr_info->op_reply_q)  in mpi3mr_isr()
687 	if (!intr_info->op_reply_q->enable_irq_poll ||  in mpi3mr_isr()
688 	    !atomic_read(&intr_info->op_reply_q->pend_ios))  in mpi3mr_isr()
691 	disable_irq_nosync(intr_info->os_irq);  in mpi3mr_isr()
697  * mpi3mr_isr_poll - Reply queue polling routine
713 	if (!intr_info || !intr_info->op_reply_q)  in mpi3mr_isr_poll()
716 	mrioc = intr_info->mrioc;  in mpi3mr_isr_poll()
717 	midx = intr_info->msix_index;  in mpi3mr_isr_poll()
721 		if (!mrioc->intr_enabled || mrioc->unrecoverable)  in mpi3mr_isr_poll()
726 		if (intr_info->op_reply_q)  in mpi3mr_isr_poll()
729 				intr_info->op_reply_q);  in mpi3mr_isr_poll()
733 	} while (atomic_read(&intr_info->op_reply_q->pend_ios) &&  in mpi3mr_isr_poll()
734 	    (num_op_reply < mrioc->max_host_ios));  in mpi3mr_isr_poll()
736 	intr_info->op_reply_q->enable_irq_poll = false;  in mpi3mr_isr_poll()
737 	enable_irq(intr_info->os_irq);  in mpi3mr_isr_poll()
745  * mpi3mr_request_irq - Request IRQ and register ISR
751  * Return: 0 on success and non zero on failures.
755 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_request_irq()
756 	struct mpi3mr_intr_info *intr_info = mrioc->intr_info + index;  in mpi3mr_request_irq()
759 	intr_info->mrioc = mrioc;  in mpi3mr_request_irq()
760 	intr_info->msix_index = index;  in mpi3mr_request_irq()
761 	intr_info->op_reply_q = NULL;  in mpi3mr_request_irq()
763 	snprintf(intr_info->name, MPI3MR_NAME_LENGTH, "%s%d-msix%d",  in mpi3mr_request_irq()
764 	    mrioc->driver_name, mrioc->id, index);  in mpi3mr_request_irq()
768 	    mpi3mr_isr_poll, IRQF_SHARED, intr_info->name, intr_info);  in mpi3mr_request_irq()
771 	    NULL, IRQF_SHARED, intr_info->name, intr_info);  in mpi3mr_request_irq()
775 		    intr_info->name, pci_irq_vector(pdev, index));  in mpi3mr_request_irq()
779 	intr_info->os_irq = pci_irq_vector(pdev, index);  in mpi3mr_request_irq()
785 	if (!mrioc->requested_poll_qcount)  in mpi3mr_calc_poll_queues()
790 		(mrioc->requested_poll_qcount < max_vectors - 2)) {  in mpi3mr_calc_poll_queues()
793 		    mrioc->requested_poll_qcount, max_vectors);  in mpi3mr_calc_poll_queues()
797 		    mrioc->requested_poll_qcount, max_vectors);  in mpi3mr_calc_poll_queues()
798 		mrioc->requested_poll_qcount = 0;  in mpi3mr_calc_poll_queues()
803  * mpi3mr_setup_isr - Setup ISR for the controller
809  * Return: 0 on success and non zero on failures.
819 	if (mrioc->is_intr_info_set)  in mpi3mr_setup_isr()
826 		retval = pci_alloc_irq_vectors(mrioc->pdev,  in mpi3mr_setup_isr()
835 		    min_t(int, mrioc->cpu_count + 1 +  in mpi3mr_setup_isr()
836 			mrioc->requested_poll_qcount, mrioc->msix_count);  in mpi3mr_setup_isr()
841 		    "MSI-X vectors supported: %d, no of cores: %d,",  in mpi3mr_setup_isr()
842 		    mrioc->msix_count, mrioc->cpu_count);  in mpi3mr_setup_isr()
844 		    "MSI-x vectors requested: %d poll_queues %d\n",  in mpi3mr_setup_isr()
845 		    max_vectors, mrioc->requested_poll_qcount);  in mpi3mr_setup_isr()
847 		desc.post_vectors = mrioc->requested_poll_qcount;  in mpi3mr_setup_isr()
851 		retval = pci_alloc_irq_vectors_affinity(mrioc->pdev,  in mpi3mr_setup_isr()
862 		 * If only one MSI-x is allocated, then MSI-x 0 will be shared  in mpi3mr_setup_isr()
863 		 * between Admin queue and operational queue  in mpi3mr_setup_isr()
866 			mrioc->op_reply_q_offset = 0;  in mpi3mr_setup_isr()
874 		mrioc->op_reply_q_offset = (max_vectors > 1) ? 1 : 0;  in mpi3mr_setup_isr()
880 	mrioc->intr_info = kzalloc(sizeof(struct mpi3mr_intr_info) * max_vectors,  in mpi3mr_setup_isr()
882 	if (!mrioc->intr_info) {  in mpi3mr_setup_isr()
883 		retval = -ENOMEM;  in mpi3mr_setup_isr()
884 		pci_free_irq_vectors(mrioc->pdev);  in mpi3mr_setup_isr()
890 			mrioc->intr_info_count = i;  in mpi3mr_setup_isr()
895 		mrioc->is_intr_info_set = true;  in mpi3mr_setup_isr()
896 	mrioc->intr_info_count = max_vectors;  in mpi3mr_setup_isr()
986  * mpi3mr_reset_rc_name - get reset reason code name
1017  * mpi3mr_reset_type_name - get reset type name
1039  * mpi3mr_is_fault_recoverable - Read fault code and decide
1048 	fault = (readl(&mrioc->sysif_regs->fault) &  in mpi3mr_is_fault_recoverable()
1069  * mpi3mr_print_fault_info - Display fault information
1081 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_print_fault_info()
1084 		code = readl(&mrioc->sysif_regs->fault);  in mpi3mr_print_fault_info()
1085 		code1 = readl(&mrioc->sysif_regs->fault_info[0]);  in mpi3mr_print_fault_info()
1086 		code2 = readl(&mrioc->sysif_regs->fault_info[1]);  in mpi3mr_print_fault_info()
1087 		code3 = readl(&mrioc->sysif_regs->fault_info[2]);  in mpi3mr_print_fault_info()
1096  * mpi3mr_get_iocstate - Get IOC State
1109 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_get_iocstate()
1110 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_get_iocstate()
1112 	if (mrioc->unrecoverable)  in mpi3mr_get_iocstate()
1131  * mpi3mr_free_ioctl_dma_memory - free memory for ioctl dma
1143 	if (!mrioc->ioctl_dma_pool)  in mpi3mr_free_ioctl_dma_memory()
1147 		mem_desc = &mrioc->ioctl_sge[i];  in mpi3mr_free_ioctl_dma_memory()
1148 		if (mem_desc->addr) {  in mpi3mr_free_ioctl_dma_memory()
1149 			dma_pool_free(mrioc->ioctl_dma_pool,  in mpi3mr_free_ioctl_dma_memory()
1150 				      mem_desc->addr,  in mpi3mr_free_ioctl_dma_memory()
1151 				      mem_desc->dma_addr);  in mpi3mr_free_ioctl_dma_memory()
1152 			mem_desc->addr = NULL;  in mpi3mr_free_ioctl_dma_memory()
1155 	dma_pool_destroy(mrioc->ioctl_dma_pool);  in mpi3mr_free_ioctl_dma_memory()
1156 	mrioc->ioctl_dma_pool = NULL;  in mpi3mr_free_ioctl_dma_memory()
1157 	mem_desc = &mrioc->ioctl_chain_sge;  in mpi3mr_free_ioctl_dma_memory()
1159 	if (mem_desc->addr) {  in mpi3mr_free_ioctl_dma_memory()
1160 		dma_free_coherent(&mrioc->pdev->dev, mem_desc->size,  in mpi3mr_free_ioctl_dma_memory()
1161 				  mem_desc->addr, mem_desc->dma_addr);  in mpi3mr_free_ioctl_dma_memory()
1162 		mem_desc->addr = NULL;  in mpi3mr_free_ioctl_dma_memory()
1164 	mem_desc = &mrioc->ioctl_resp_sge;  in mpi3mr_free_ioctl_dma_memory()
1165 	if (mem_desc->addr) {  in mpi3mr_free_ioctl_dma_memory()
1166 		dma_free_coherent(&mrioc->pdev->dev, mem_desc->size,  in mpi3mr_free_ioctl_dma_memory()
1167 				  mem_desc->addr, mem_desc->dma_addr);  in mpi3mr_free_ioctl_dma_memory()
1168 		mem_desc->addr = NULL;  in mpi3mr_free_ioctl_dma_memory()
1171 	mrioc->ioctl_sges_allocated = false;  in mpi3mr_free_ioctl_dma_memory()
1175  * mpi3mr_alloc_ioctl_dma_memory - Alloc memory for ioctl dma
1189 	mrioc->ioctl_dma_pool = dma_pool_create("ioctl dma pool",  in mpi3mr_alloc_ioctl_dma_memory()
1190 						&mrioc->pdev->dev,  in mpi3mr_alloc_ioctl_dma_memory()
1194 	if (!mrioc->ioctl_dma_pool) {  in mpi3mr_alloc_ioctl_dma_memory()
1200 		mem_desc = &mrioc->ioctl_sge[i];  in mpi3mr_alloc_ioctl_dma_memory()
1201 		mem_desc->size = MPI3MR_IOCTL_SGE_SIZE;  in mpi3mr_alloc_ioctl_dma_memory()
1202 		mem_desc->addr = dma_pool_zalloc(mrioc->ioctl_dma_pool,  in mpi3mr_alloc_ioctl_dma_memory()
1204 						 &mem_desc->dma_addr);  in mpi3mr_alloc_ioctl_dma_memory()
1205 		if (!mem_desc->addr)  in mpi3mr_alloc_ioctl_dma_memory()
1209 	mem_desc = &mrioc->ioctl_chain_sge;  in mpi3mr_alloc_ioctl_dma_memory()
1210 	mem_desc->size = MPI3MR_PAGE_SIZE_4K;  in mpi3mr_alloc_ioctl_dma_memory()
1211 	mem_desc->addr = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_ioctl_dma_memory()
1212 					    mem_desc->size,  in mpi3mr_alloc_ioctl_dma_memory()
1213 					    &mem_desc->dma_addr,  in mpi3mr_alloc_ioctl_dma_memory()
1215 	if (!mem_desc->addr)  in mpi3mr_alloc_ioctl_dma_memory()
1218 	mem_desc = &mrioc->ioctl_resp_sge;  in mpi3mr_alloc_ioctl_dma_memory()
1219 	mem_desc->size = MPI3MR_PAGE_SIZE_4K;  in mpi3mr_alloc_ioctl_dma_memory()
1220 	mem_desc->addr = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_ioctl_dma_memory()
1221 					    mem_desc->size,  in mpi3mr_alloc_ioctl_dma_memory()
1222 					    &mem_desc->dma_addr,  in mpi3mr_alloc_ioctl_dma_memory()
1224 	if (!mem_desc->addr)  in mpi3mr_alloc_ioctl_dma_memory()
1227 	mrioc->ioctl_sges_allocated = true;  in mpi3mr_alloc_ioctl_dma_memory()
1237  * mpi3mr_clear_reset_history - clear reset history
1249 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_clear_reset_history()
1251 		writel(ioc_status, &mrioc->sysif_regs->ioc_status);  in mpi3mr_clear_reset_history()
1255  * mpi3mr_issue_and_process_mur - Message unit Reset handler
1262  * Return: 0 on success, -1 on failure.
1268 	int retval = -1;  in mpi3mr_issue_and_process_mur()
1271 	if (mrioc->unrecoverable) {  in mpi3mr_issue_and_process_mur()
1278 			(mrioc->facts.ioc_num <<  in mpi3mr_issue_and_process_mur()
1280 	writel(scratch_pad0, &mrioc->sysif_regs->scratchpad[0]);  in mpi3mr_issue_and_process_mur()
1281 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_and_process_mur()
1283 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_and_process_mur()
1287 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_and_process_mur()
1297 	} while (--timeout);  in mpi3mr_issue_and_process_mur()
1299 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_and_process_mur()
1311  * mpi3mr_revalidate_factsdata - validate IOCFacts parameters
1316  * older values else return -EPERM
1323 	if (mrioc->facts.reply_sz > mrioc->reply_sz) {  in mpi3mr_revalidate_factsdata()
1326 		    mrioc->reply_sz, mrioc->facts.reply_sz);  in mpi3mr_revalidate_factsdata()
1327 		return -EPERM;  in mpi3mr_revalidate_factsdata()
1330 	if (mrioc->facts.max_op_reply_q < mrioc->num_op_reply_q) {  in mpi3mr_revalidate_factsdata()
1332 		    "cannot reduce number of operational reply queues from %d to %d\n",  in mpi3mr_revalidate_factsdata()
1333 		    mrioc->num_op_reply_q,  in mpi3mr_revalidate_factsdata()
1334 		    mrioc->facts.max_op_reply_q);  in mpi3mr_revalidate_factsdata()
1335 		return -EPERM;  in mpi3mr_revalidate_factsdata()
1338 	if (mrioc->facts.max_op_req_q < mrioc->num_op_req_q) {  in mpi3mr_revalidate_factsdata()
1340 		    "cannot reduce number of operational request queues from %d to %d\n",  in mpi3mr_revalidate_factsdata()
1341 		    mrioc->num_op_req_q, mrioc->facts.max_op_req_q);  in mpi3mr_revalidate_factsdata()
1342 		return -EPERM;  in mpi3mr_revalidate_factsdata()
1345 	if (mrioc->shost->max_sectors != (mrioc->facts.max_data_length / 512))  in mpi3mr_revalidate_factsdata()
1349 			    mrioc->shost->max_sectors * 512, mrioc->facts.max_data_length);  in mpi3mr_revalidate_factsdata()
1351 	if ((mrioc->sas_transport_enabled) && (mrioc->facts.ioc_capabilities &  in mpi3mr_revalidate_factsdata()
1358 	if (mrioc->facts.max_devhandle > mrioc->dev_handle_bitmap_bits) {  in mpi3mr_revalidate_factsdata()
1359 		removepend_bitmap = bitmap_zalloc(mrioc->facts.max_devhandle,  in mpi3mr_revalidate_factsdata()
1364 				mrioc->dev_handle_bitmap_bits,  in mpi3mr_revalidate_factsdata()
1365 				mrioc->facts.max_devhandle);  in mpi3mr_revalidate_factsdata()
1366 			return -EPERM;  in mpi3mr_revalidate_factsdata()
1368 		bitmap_free(mrioc->removepend_bitmap);  in mpi3mr_revalidate_factsdata()
1369 		mrioc->removepend_bitmap = removepend_bitmap;  in mpi3mr_revalidate_factsdata()
1372 			 mrioc->dev_handle_bitmap_bits,  in mpi3mr_revalidate_factsdata()
1373 			 mrioc->facts.max_devhandle);  in mpi3mr_revalidate_factsdata()
1374 		mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;  in mpi3mr_revalidate_factsdata()
1381  * mpi3mr_bring_ioc_ready - Bring controller to ready state
1400 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_bring_ioc_ready()
1401 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_bring_ioc_ready()
1402 	base_info = lo_hi_readq(&mrioc->sysif_regs->ioc_information);  in mpi3mr_bring_ioc_ready()
1407 		mrioc->unrecoverable = 1;  in mpi3mr_bring_ioc_ready()
1412 	mrioc->ready_timeout =  in mpi3mr_bring_ioc_ready()
1416 	ioc_info(mrioc, "ready timeout: %d seconds\n", mrioc->ready_timeout);  in mpi3mr_bring_ioc_ready()
1422 	timeout = mrioc->ready_timeout * 10;  in mpi3mr_bring_ioc_ready()
1431 		if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_bring_ioc_ready()
1432 			mrioc->unrecoverable = 1;  in mpi3mr_bring_ioc_ready()
1438 	} while (--timeout);  in mpi3mr_bring_ioc_ready()
1456 					readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_bring_ioc_ready()
1460 				if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_bring_ioc_ready()
1461 					mrioc->unrecoverable = 1;  in mpi3mr_bring_ioc_ready()
1466 			} while (--timeout);  in mpi3mr_bring_ioc_ready()
1495 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_bring_ioc_ready()
1497 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_bring_ioc_ready()
1502 	timeout = mrioc->ready_timeout * 10;  in mpi3mr_bring_ioc_ready()
1511 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_bring_ioc_ready()
1517 		if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_bring_ioc_ready()
1518 			mrioc->unrecoverable = 1;  in mpi3mr_bring_ioc_ready()
1521 			retval = -1;  in mpi3mr_bring_ioc_ready()
1525 		elapsed_time_sec = jiffies_to_msecs(jiffies - start_time)/1000;  in mpi3mr_bring_ioc_ready()
1526 	} while (elapsed_time_sec < mrioc->ready_timeout);  in mpi3mr_bring_ioc_ready()
1529 	elapsed_time_sec = jiffies_to_msecs(jiffies - start_time)/1000;  in mpi3mr_bring_ioc_ready()
1530 	if ((retry < 2) && (elapsed_time_sec < (mrioc->ready_timeout - 60))) {  in mpi3mr_bring_ioc_ready()
1547  * mpi3mr_soft_reset_success - Check softreset is success or not
1566  * mpi3mr_diagfault_success - Check diag fault is success or not
1581 	fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;  in mpi3mr_diagfault_success()
1590  * mpi3mr_set_diagsave - Set diag save bit for snapdump
1602 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_set_diagsave()
1604 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_set_diagsave()
1608  * mpi3mr_issue_reset - Issue reset to the controller
1618  * Return: 0 on success, non-zero on failure.
1623 	int retval = -1;  in mpi3mr_issue_reset()
1631 	if (mrioc->unrecoverable)  in mpi3mr_issue_reset()
1652 			mrioc->unrecoverable = 1;  in mpi3mr_issue_reset()
1657 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1659 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1661 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1663 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1665 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1667 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1669 		    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1671 		host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_issue_reset()
1678 	    MPI3MR_RESET_REASON_OSTYPE_SHIFT) | (mrioc->facts.ioc_num <<  in mpi3mr_issue_reset()
1680 	writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);  in mpi3mr_issue_reset()
1682 	    &mrioc->sysif_regs->host_diagnostic);  in mpi3mr_issue_reset()
1686 			ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_reset()
1688 			    readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_reset()
1697 		} while (--timeout);  in mpi3mr_issue_reset()
1702 			ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_reset()
1708 		} while (--timeout);  in mpi3mr_issue_reset()
1715 	    &mrioc->sysif_regs->write_sequence);  in mpi3mr_issue_reset()
1717 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_reset()
1718 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_reset()
1724 		mrioc->unrecoverable = 1;  in mpi3mr_issue_reset()
1729  * mpi3mr_admin_request_post - Post request to admin queue
1739  * Return: 0 on success, non-zero on failure.
1749 	if (mrioc->unrecoverable) {  in mpi3mr_admin_request_post()
1751 		return -EFAULT;  in mpi3mr_admin_request_post()
1754 	spin_lock_irqsave(&mrioc->admin_req_lock, flags);  in mpi3mr_admin_request_post()
1755 	areq_pi = mrioc->admin_req_pi;  in mpi3mr_admin_request_post()
1756 	areq_ci = mrioc->admin_req_ci;  in mpi3mr_admin_request_post()
1757 	max_entries = mrioc->num_admin_req;  in mpi3mr_admin_request_post()
1759 	    (areq_pi == (max_entries - 1)))) {  in mpi3mr_admin_request_post()
1761 		retval = -EAGAIN;  in mpi3mr_admin_request_post()
1764 	if (!ignore_reset && mrioc->reset_in_progress) {  in mpi3mr_admin_request_post()
1766 		retval = -EAGAIN;  in mpi3mr_admin_request_post()
1769 	if (mrioc->pci_err_recovery) {  in mpi3mr_admin_request_post()
1771 		retval = -EAGAIN;  in mpi3mr_admin_request_post()
1775 	areq_entry = (u8 *)mrioc->admin_req_base +  in mpi3mr_admin_request_post()
1782 	mrioc->admin_req_pi = areq_pi;  in mpi3mr_admin_request_post()
1784 	writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);  in mpi3mr_admin_request_post()
1787 	spin_unlock_irqrestore(&mrioc->admin_req_lock, flags);  in mpi3mr_admin_request_post()
1793  * mpi3mr_free_op_req_q_segments - free request memory segments
1795  * @q_idx: operational request queue index
1797  * Free memory segments allocated for operational request queue
1807 	segments = mrioc->req_qinfo[q_idx].q_segments;  in mpi3mr_free_op_req_q_segments()
1811 	if (mrioc->enable_segqueue) {  in mpi3mr_free_op_req_q_segments()
1813 		if (mrioc->req_qinfo[q_idx].q_segment_list) {  in mpi3mr_free_op_req_q_segments()
1814 			dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_req_q_segments()
1816 			    mrioc->req_qinfo[q_idx].q_segment_list,  in mpi3mr_free_op_req_q_segments()
1817 			    mrioc->req_qinfo[q_idx].q_segment_list_dma);  in mpi3mr_free_op_req_q_segments()
1818 			mrioc->req_qinfo[q_idx].q_segment_list = NULL;  in mpi3mr_free_op_req_q_segments()
1821 		size = mrioc->req_qinfo[q_idx].segment_qd *  in mpi3mr_free_op_req_q_segments()
1822 		    mrioc->facts.op_req_sz;  in mpi3mr_free_op_req_q_segments()
1824 	for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {  in mpi3mr_free_op_req_q_segments()
1827 		dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_req_q_segments()
1831 	kfree(mrioc->req_qinfo[q_idx].q_segments);  in mpi3mr_free_op_req_q_segments()
1832 	mrioc->req_qinfo[q_idx].q_segments = NULL;  in mpi3mr_free_op_req_q_segments()
1833 	mrioc->req_qinfo[q_idx].qid = 0;  in mpi3mr_free_op_req_q_segments()
1837  * mpi3mr_free_op_reply_q_segments - free reply memory segments
1839  * @q_idx: operational reply queue index
1841  * Free memory segments allocated for operational reply queue
1851 	segments = mrioc->op_reply_qinfo[q_idx].q_segments;  in mpi3mr_free_op_reply_q_segments()
1855 	if (mrioc->enable_segqueue) {  in mpi3mr_free_op_reply_q_segments()
1857 		if (mrioc->op_reply_qinfo[q_idx].q_segment_list) {  in mpi3mr_free_op_reply_q_segments()
1858 			dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_reply_q_segments()
1860 			    mrioc->op_reply_qinfo[q_idx].q_segment_list,  in mpi3mr_free_op_reply_q_segments()
1861 			    mrioc->op_reply_qinfo[q_idx].q_segment_list_dma);  in mpi3mr_free_op_reply_q_segments()
1862 			mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;  in mpi3mr_free_op_reply_q_segments()
1865 		size = mrioc->op_reply_qinfo[q_idx].segment_qd *  in mpi3mr_free_op_reply_q_segments()
1866 		    mrioc->op_reply_desc_sz;  in mpi3mr_free_op_reply_q_segments()
1868 	for (j = 0; j < mrioc->op_reply_qinfo[q_idx].num_segments; j++) {  in mpi3mr_free_op_reply_q_segments()
1871 		dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_op_reply_q_segments()
1876 	kfree(mrioc->op_reply_qinfo[q_idx].q_segments);  in mpi3mr_free_op_reply_q_segments()
1877 	mrioc->op_reply_qinfo[q_idx].q_segments = NULL;  in mpi3mr_free_op_reply_q_segments()
1878 	mrioc->op_reply_qinfo[q_idx].qid = 0;  in mpi3mr_free_op_reply_q_segments()
1882  * mpi3mr_delete_op_reply_q - delete operational reply queue
1884  * @qidx: operational reply queue index
1889  * Return:  0 on success, non-zero on failure.
1894 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_delete_op_reply_q()
1898 	reply_qid = op_reply_q->qid;  in mpi3mr_delete_op_reply_q()
1900 	midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);  in mpi3mr_delete_op_reply_q()
1903 		retval = -1;  in mpi3mr_delete_op_reply_q()
1908 	(op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount-- :  in mpi3mr_delete_op_reply_q()
1909 	    mrioc->active_poll_qcount--;  in mpi3mr_delete_op_reply_q()
1912 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_delete_op_reply_q()
1913 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_delete_op_reply_q()
1914 		retval = -1;  in mpi3mr_delete_op_reply_q()
1916 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_delete_op_reply_q()
1919 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_delete_op_reply_q()
1920 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_delete_op_reply_q()
1921 	mrioc->init_cmds.callback = NULL;  in mpi3mr_delete_op_reply_q()
1926 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_delete_op_reply_q()
1933 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_delete_op_reply_q()
1935 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_delete_op_reply_q()
1939 		retval = -1;  in mpi3mr_delete_op_reply_q()
1942 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_delete_op_reply_q()
1946 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_delete_op_reply_q()
1947 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_delete_op_reply_q()
1948 		retval = -1;  in mpi3mr_delete_op_reply_q()
1951 	mrioc->intr_info[midx].op_reply_q = NULL;  in mpi3mr_delete_op_reply_q()
1955 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_delete_op_reply_q()
1956 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_delete_op_reply_q()
1963  * mpi3mr_alloc_op_reply_q_segments -Alloc segmented reply pool
1967  * Allocate segmented memory pools for operational reply
1970  * Return: 0 on success, non-zero on failure.
1974 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_alloc_op_reply_q_segments()
1979 	if (mrioc->enable_segqueue) {  in mpi3mr_alloc_op_reply_q_segments()
1980 		op_reply_q->segment_qd =  in mpi3mr_alloc_op_reply_q_segments()
1981 		    MPI3MR_OP_REP_Q_SEG_SIZE / mrioc->op_reply_desc_sz;  in mpi3mr_alloc_op_reply_q_segments()
1985 		op_reply_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_reply_q_segments()
1986 		    MPI3MR_MAX_SEG_LIST_SIZE, &op_reply_q->q_segment_list_dma,  in mpi3mr_alloc_op_reply_q_segments()
1988 		if (!op_reply_q->q_segment_list)  in mpi3mr_alloc_op_reply_q_segments()
1989 			return -ENOMEM;  in mpi3mr_alloc_op_reply_q_segments()
1990 		q_segment_list_entry = (u64 *)op_reply_q->q_segment_list;  in mpi3mr_alloc_op_reply_q_segments()
1992 		op_reply_q->segment_qd = op_reply_q->num_replies;  in mpi3mr_alloc_op_reply_q_segments()
1993 		size = op_reply_q->num_replies * mrioc->op_reply_desc_sz;  in mpi3mr_alloc_op_reply_q_segments()
1996 	op_reply_q->num_segments = DIV_ROUND_UP(op_reply_q->num_replies,  in mpi3mr_alloc_op_reply_q_segments()
1997 	    op_reply_q->segment_qd);  in mpi3mr_alloc_op_reply_q_segments()
1999 	op_reply_q->q_segments = kcalloc(op_reply_q->num_segments,  in mpi3mr_alloc_op_reply_q_segments()
2001 	if (!op_reply_q->q_segments)  in mpi3mr_alloc_op_reply_q_segments()
2002 		return -ENOMEM;  in mpi3mr_alloc_op_reply_q_segments()
2004 	segments = op_reply_q->q_segments;  in mpi3mr_alloc_op_reply_q_segments()
2005 	for (i = 0; i < op_reply_q->num_segments; i++) {  in mpi3mr_alloc_op_reply_q_segments()
2007 		    dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_reply_q_segments()
2010 			return -ENOMEM;  in mpi3mr_alloc_op_reply_q_segments()
2011 		if (mrioc->enable_segqueue)  in mpi3mr_alloc_op_reply_q_segments()
2020  * mpi3mr_alloc_op_req_q_segments - Alloc segmented req pool.
2024  * Allocate segmented memory pools for operational request
2027  * Return: 0 on success, non-zero on failure.
2031 	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;  in mpi3mr_alloc_op_req_q_segments()
2036 	if (mrioc->enable_segqueue) {  in mpi3mr_alloc_op_req_q_segments()
2037 		op_req_q->segment_qd =  in mpi3mr_alloc_op_req_q_segments()
2038 		    MPI3MR_OP_REQ_Q_SEG_SIZE / mrioc->facts.op_req_sz;  in mpi3mr_alloc_op_req_q_segments()
2042 		op_req_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_req_q_segments()
2043 		    MPI3MR_MAX_SEG_LIST_SIZE, &op_req_q->q_segment_list_dma,  in mpi3mr_alloc_op_req_q_segments()
2045 		if (!op_req_q->q_segment_list)  in mpi3mr_alloc_op_req_q_segments()
2046 			return -ENOMEM;  in mpi3mr_alloc_op_req_q_segments()
2047 		q_segment_list_entry = (u64 *)op_req_q->q_segment_list;  in mpi3mr_alloc_op_req_q_segments()
2050 		op_req_q->segment_qd = op_req_q->num_requests;  in mpi3mr_alloc_op_req_q_segments()
2051 		size = op_req_q->num_requests * mrioc->facts.op_req_sz;  in mpi3mr_alloc_op_req_q_segments()
2054 	op_req_q->num_segments = DIV_ROUND_UP(op_req_q->num_requests,  in mpi3mr_alloc_op_req_q_segments()
2055 	    op_req_q->segment_qd);  in mpi3mr_alloc_op_req_q_segments()
2057 	op_req_q->q_segments = kcalloc(op_req_q->num_segments,  in mpi3mr_alloc_op_req_q_segments()
2059 	if (!op_req_q->q_segments)  in mpi3mr_alloc_op_req_q_segments()
2060 		return -ENOMEM;  in mpi3mr_alloc_op_req_q_segments()
2062 	segments = op_req_q->q_segments;  in mpi3mr_alloc_op_req_q_segments()
2063 	for (i = 0; i < op_req_q->num_segments; i++) {  in mpi3mr_alloc_op_req_q_segments()
2065 		    dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_alloc_op_req_q_segments()
2068 			return -ENOMEM;  in mpi3mr_alloc_op_req_q_segments()
2069 		if (mrioc->enable_segqueue)  in mpi3mr_alloc_op_req_q_segments()
2078  * mpi3mr_create_op_reply_q - create operational reply queue
2080  * @qidx: operational reply queue index
2085  * Return:  0 on success, non-zero on failure.
2090 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_create_op_reply_q()
2094 	reply_qid = op_reply_q->qid;  in mpi3mr_create_op_reply_q()
2096 	midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);  in mpi3mr_create_op_reply_q()
2099 		retval = -1;  in mpi3mr_create_op_reply_q()
2108 	if (mrioc->pdev->device == MPI3_MFGPAGE_DEVID_SAS4116) {  in mpi3mr_create_op_reply_q()
2109 		if (mrioc->pdev->revision)  in mpi3mr_create_op_reply_q()
2110 			op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD;  in mpi3mr_create_op_reply_q()
2112 			op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD4K;  in mpi3mr_create_op_reply_q()
2114 		op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD2K;  in mpi3mr_create_op_reply_q()
2116 	op_reply_q->ci = 0;  in mpi3mr_create_op_reply_q()
2117 	op_reply_q->ephase = 1;  in mpi3mr_create_op_reply_q()
2118 	atomic_set(&op_reply_q->pend_ios, 0);  in mpi3mr_create_op_reply_q()
2119 	atomic_set(&op_reply_q->in_use, 0);  in mpi3mr_create_op_reply_q()
2120 	op_reply_q->enable_irq_poll = false;  in mpi3mr_create_op_reply_q()
2121 	op_reply_q->qfull_watermark =  in mpi3mr_create_op_reply_q()
2122 		op_reply_q->num_replies - (MPI3MR_THRESHOLD_REPLY_COUNT * 2);  in mpi3mr_create_op_reply_q()
2124 	if (!op_reply_q->q_segments) {  in mpi3mr_create_op_reply_q()
2133 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_reply_q()
2134 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_create_op_reply_q()
2135 		retval = -1;  in mpi3mr_create_op_reply_q()
2139 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_create_op_reply_q()
2140 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_create_op_reply_q()
2141 	mrioc->init_cmds.callback = NULL;  in mpi3mr_create_op_reply_q()
2146 	if (midx < (mrioc->intr_info_count - mrioc->requested_poll_qcount))  in mpi3mr_create_op_reply_q()
2147 		op_reply_q->qtype = MPI3MR_DEFAULT_QUEUE;  in mpi3mr_create_op_reply_q()
2149 		op_reply_q->qtype = MPI3MR_POLL_QUEUE;  in mpi3mr_create_op_reply_q()
2151 	if (op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) {  in mpi3mr_create_op_reply_q()
2155 			cpu_to_le16(mrioc->intr_info[midx].msix_index);  in mpi3mr_create_op_reply_q()
2157 		create_req.msix_index = cpu_to_le16(mrioc->intr_info_count - 1);  in mpi3mr_create_op_reply_q()
2160 		if (!mrioc->active_poll_qcount)  in mpi3mr_create_op_reply_q()
2161 			disable_irq_nosync(pci_irq_vector(mrioc->pdev,  in mpi3mr_create_op_reply_q()
2162 			    mrioc->intr_info_count - 1));  in mpi3mr_create_op_reply_q()
2165 	if (mrioc->enable_segqueue) {  in mpi3mr_create_op_reply_q()
2169 		    op_reply_q->q_segment_list_dma);  in mpi3mr_create_op_reply_q()
2172 		    op_reply_q->q_segments[0].segment_dma);  in mpi3mr_create_op_reply_q()
2174 	create_req.size = cpu_to_le16(op_reply_q->num_replies);  in mpi3mr_create_op_reply_q()
2176 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_create_op_reply_q()
2183 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_create_op_reply_q()
2185 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_create_op_reply_q()
2189 		retval = -1;  in mpi3mr_create_op_reply_q()
2192 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_create_op_reply_q()
2196 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_create_op_reply_q()
2197 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_create_op_reply_q()
2198 		retval = -1;  in mpi3mr_create_op_reply_q()
2201 	op_reply_q->qid = reply_qid;  in mpi3mr_create_op_reply_q()
2202 	if (midx < mrioc->intr_info_count)  in mpi3mr_create_op_reply_q()
2203 		mrioc->intr_info[midx].op_reply_q = op_reply_q;  in mpi3mr_create_op_reply_q()
2205 	(op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount++ :  in mpi3mr_create_op_reply_q()
2206 	    mrioc->active_poll_qcount++;  in mpi3mr_create_op_reply_q()
2209 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_create_op_reply_q()
2210 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_reply_q()
2217  * mpi3mr_create_op_req_q - create operational request queue
2219  * @idx: operational request queue index
2225  * Return:  0 on success, non-zero on failure.
2231 	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + idx;  in mpi3mr_create_op_req_q()
2235 	req_qid = op_req_q->qid;  in mpi3mr_create_op_req_q()
2238 		retval = -1;  in mpi3mr_create_op_req_q()
2246 	op_req_q->num_requests = MPI3MR_OP_REQ_Q_QD;  in mpi3mr_create_op_req_q()
2247 	op_req_q->ci = 0;  in mpi3mr_create_op_req_q()
2248 	op_req_q->pi = 0;  in mpi3mr_create_op_req_q()
2249 	op_req_q->reply_qid = reply_qid;  in mpi3mr_create_op_req_q()
2250 	spin_lock_init(&op_req_q->q_lock);  in mpi3mr_create_op_req_q()
2252 	if (!op_req_q->q_segments) {  in mpi3mr_create_op_req_q()
2261 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_req_q()
2262 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_create_op_req_q()
2263 		retval = -1;  in mpi3mr_create_op_req_q()
2267 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_create_op_req_q()
2268 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_create_op_req_q()
2269 	mrioc->init_cmds.callback = NULL;  in mpi3mr_create_op_req_q()
2273 	if (mrioc->enable_segqueue) {  in mpi3mr_create_op_req_q()
2277 		    op_req_q->q_segment_list_dma);  in mpi3mr_create_op_req_q()
2280 		    op_req_q->q_segments[0].segment_dma);  in mpi3mr_create_op_req_q()
2282 	create_req.size = cpu_to_le16(op_req_q->num_requests);  in mpi3mr_create_op_req_q()
2284 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_create_op_req_q()
2291 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_create_op_req_q()
2293 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_create_op_req_q()
2297 		retval = -1;  in mpi3mr_create_op_req_q()
2300 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_create_op_req_q()
2304 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_create_op_req_q()
2305 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_create_op_req_q()
2306 		retval = -1;  in mpi3mr_create_op_req_q()
2309 	op_req_q->qid = req_qid;  in mpi3mr_create_op_req_q()
2312 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_create_op_req_q()
2313 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_create_op_req_q()
2320  * mpi3mr_create_op_queues - create operational queue pairs
2323  * Allocate memory for operational queue meta data and call
2326  * Return: 0 on success, non-zero on failures.
2333 	num_queues = min_t(int, mrioc->facts.max_op_reply_q,  in mpi3mr_create_op_queues()
2334 	    mrioc->facts.max_op_req_q);  in mpi3mr_create_op_queues()
2337 	    mrioc->intr_info_count - mrioc->op_reply_q_offset;  in mpi3mr_create_op_queues()
2338 	if (!mrioc->num_queues)  in mpi3mr_create_op_queues()
2339 		mrioc->num_queues = min_t(int, num_queues, msix_count_op_q);  in mpi3mr_create_op_queues()
2344 	num_queues = mrioc->num_op_reply_q ?  in mpi3mr_create_op_queues()
2345 	    mrioc->num_op_reply_q : mrioc->num_queues;  in mpi3mr_create_op_queues()
2346 	ioc_info(mrioc, "trying to create %d operational queue pairs\n",  in mpi3mr_create_op_queues()
2349 	if (!mrioc->req_qinfo) {  in mpi3mr_create_op_queues()
2350 		mrioc->req_qinfo = kcalloc(num_queues,  in mpi3mr_create_op_queues()
2352 		if (!mrioc->req_qinfo) {  in mpi3mr_create_op_queues()
2353 			retval = -1;  in mpi3mr_create_op_queues()
2357 		mrioc->op_reply_qinfo = kzalloc(sizeof(struct op_reply_qinfo) *  in mpi3mr_create_op_queues()
2359 		if (!mrioc->op_reply_qinfo) {  in mpi3mr_create_op_queues()
2360 			retval = -1;  in mpi3mr_create_op_queues()
2365 	if (mrioc->enable_segqueue)  in mpi3mr_create_op_queues()
2367 		    "allocating operational queues through segmented queues\n");  in mpi3mr_create_op_queues()
2375 		    mrioc->op_reply_qinfo[i].qid)) {  in mpi3mr_create_op_queues()
2384 		retval = -1;  in mpi3mr_create_op_queues()
2387 	mrioc->num_op_reply_q = mrioc->num_op_req_q = i;  in mpi3mr_create_op_queues()
2389 	    "successfully created %d operational queue pairs(default/polled) queue = (%d/%d)\n",  in mpi3mr_create_op_queues()
2390 	    mrioc->num_op_reply_q, mrioc->default_qcount,  in mpi3mr_create_op_queues()
2391 	    mrioc->active_poll_qcount);  in mpi3mr_create_op_queues()
2395 	kfree(mrioc->req_qinfo);  in mpi3mr_create_op_queues()
2396 	mrioc->req_qinfo = NULL;  in mpi3mr_create_op_queues()
2398 	kfree(mrioc->op_reply_qinfo);  in mpi3mr_create_op_queues()
2399 	mrioc->op_reply_qinfo = NULL;  in mpi3mr_create_op_queues()
2405  * mpi3mr_op_request_post - Post request to operational queue
2407  * @op_req_q: Operational request queue info
2410  * Post the MPI3 request into operational request queue and
2414  * Return: 0 on success, non-zero on failure.
2424 	u16 req_sz = mrioc->facts.op_req_sz;  in mpi3mr_op_request_post()
2425 	struct segments *segments = op_req_q->q_segments;  in mpi3mr_op_request_post()
2428 	reply_qidx = op_req_q->reply_qid - 1;  in mpi3mr_op_request_post()
2429 	op_reply_q = mrioc->op_reply_qinfo + reply_qidx;  in mpi3mr_op_request_post()
2431 	if (mrioc->unrecoverable)  in mpi3mr_op_request_post()
2432 		return -EFAULT;  in mpi3mr_op_request_post()
2434 	spin_lock_irqsave(&op_req_q->q_lock, flags);  in mpi3mr_op_request_post()
2435 	pi = op_req_q->pi;  in mpi3mr_op_request_post()
2436 	max_entries = op_req_q->num_requests;  in mpi3mr_op_request_post()
2440 		    reply_qidx, mrioc->op_reply_q_offset);  in mpi3mr_op_request_post()
2441 		mpi3mr_process_op_reply_q(mrioc, mrioc->intr_info[midx].op_reply_q);  in mpi3mr_op_request_post()
2444 			retval = -EAGAIN;  in mpi3mr_op_request_post()
2449 	if (mrioc->reset_in_progress) {  in mpi3mr_op_request_post()
2451 		retval = -EAGAIN;  in mpi3mr_op_request_post()
2454 	if (mrioc->pci_err_recovery) {  in mpi3mr_op_request_post()
2455 …ioc_err(mrioc, "operational request queue submission failed due to pci error recovery in progress\…  in mpi3mr_op_request_post()
2456 		retval = -EAGAIN;  in mpi3mr_op_request_post()
2461 	if ((mrioc->prevent_reply_qfull == true) &&  in mpi3mr_op_request_post()
2462 		(atomic_read(&op_reply_q->pend_ios) >  in mpi3mr_op_request_post()
2463 	     (op_reply_q->qfull_watermark))) {  in mpi3mr_op_request_post()
2464 		atomic_inc(&mrioc->reply_qfull_count);  in mpi3mr_op_request_post()
2465 		retval = -EAGAIN;  in mpi3mr_op_request_post()
2469 	segment_base_addr = segments[pi / op_req_q->segment_qd].segment;  in mpi3mr_op_request_post()
2471 	    ((pi % op_req_q->segment_qd) * req_sz);  in mpi3mr_op_request_post()
2478 	op_req_q->pi = pi;  in mpi3mr_op_request_post()
2481 	if (atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios)  in mpi3mr_op_request_post()
2483 		mrioc->op_reply_qinfo[reply_qidx].enable_irq_poll = true;  in mpi3mr_op_request_post()
2485 	atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios);  in mpi3mr_op_request_post()
2488 	writel(op_req_q->pi,  in mpi3mr_op_request_post()
2489 	    &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].producer_index);  in mpi3mr_op_request_post()
2492 	spin_unlock_irqrestore(&op_req_q->q_lock, flags);  in mpi3mr_op_request_post()
2497  * mpi3mr_check_rh_fault_ioc - check reset history and fault
2515 	if (mrioc->unrecoverable) {  in mpi3mr_check_rh_fault_ioc()
2520 	if (!pci_device_is_present(mrioc->pdev)) {  in mpi3mr_check_rh_fault_ioc()
2521 		mrioc->unrecoverable = 1;  in mpi3mr_check_rh_fault_ioc()
2526 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_check_rh_fault_ioc()
2533 		trigger_data.fault = (readl(&mrioc->sysif_regs->fault) &  in mpi3mr_check_rh_fault_ioc()
2545 	trigger_data.fault = (readl(&mrioc->sysif_regs->fault) &  in mpi3mr_check_rh_fault_ioc()
2551 		host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_check_rh_fault_ioc()
2555 	} while (--timeout);  in mpi3mr_check_rh_fault_ioc()
2559  * mpi3mr_sync_timestamp - Issue time stamp sync request
2565  * Return: 0 on success, non-zero on failure.
2574 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_sync_timestamp()
2575 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_sync_timestamp()
2576 		retval = -1;  in mpi3mr_sync_timestamp()
2578 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_sync_timestamp()
2581 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_sync_timestamp()
2582 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_sync_timestamp()
2583 	mrioc->init_cmds.callback = NULL;  in mpi3mr_sync_timestamp()
2590 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_sync_timestamp()
2598 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_sync_timestamp()
2600 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_sync_timestamp()
2602 		mrioc->init_cmds.is_waiting = 0;  in mpi3mr_sync_timestamp()
2603 		if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))  in mpi3mr_sync_timestamp()
2606 		retval = -1;  in mpi3mr_sync_timestamp()
2609 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_sync_timestamp()
2613 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_sync_timestamp()
2614 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_sync_timestamp()
2615 		retval = -1;  in mpi3mr_sync_timestamp()
2620 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_sync_timestamp()
2621 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_sync_timestamp()
2628  * mpi3mr_print_pkg_ver - display controller fw package version
2634  * Return: 0 on success and non-zero on failure.
2639 	int retval = -1;  in mpi3mr_print_pkg_ver()
2646 	data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,  in mpi3mr_print_pkg_ver()
2649 		return -ENOMEM;  in mpi3mr_print_pkg_ver()
2652 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_print_pkg_ver()
2653 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_print_pkg_ver()
2655 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_print_pkg_ver()
2658 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_print_pkg_ver()
2659 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_print_pkg_ver()
2660 	mrioc->init_cmds.callback = NULL;  in mpi3mr_print_pkg_ver()
2670 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_print_pkg_ver()
2677 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_print_pkg_ver()
2679 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_print_pkg_ver()
2683 		retval = -1;  in mpi3mr_print_pkg_ver()
2686 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_print_pkg_ver()
2689 		if (manifest->manifest_type == MPI3_CI_MANIFEST_TYPE_MPI) {  in mpi3mr_print_pkg_ver()
2691 			    "firmware package version(%d.%d.%d.%d.%05d-%05d)\n",  in mpi3mr_print_pkg_ver()
2692 			    manifest->package_version.gen_major,  in mpi3mr_print_pkg_ver()
2693 			    manifest->package_version.gen_minor,  in mpi3mr_print_pkg_ver()
2694 			    manifest->package_version.phase_major,  in mpi3mr_print_pkg_ver()
2695 			    manifest->package_version.phase_minor,  in mpi3mr_print_pkg_ver()
2696 			    manifest->package_version.customer_id,  in mpi3mr_print_pkg_ver()
2697 			    manifest->package_version.build_num);  in mpi3mr_print_pkg_ver()
2702 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_print_pkg_ver()
2703 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_print_pkg_ver()
2707 		dma_free_coherent(&mrioc->pdev->dev, data_len, data,  in mpi3mr_print_pkg_ver()
2713  * mpi3mr_watchdog_work - watchdog thread to monitor faults
2732 	if (mrioc->reset_in_progress || mrioc->pci_err_recovery)  in mpi3mr_watchdog_work()
2735 	if (!mrioc->unrecoverable && !pci_device_is_present(mrioc->pdev)) {  in mpi3mr_watchdog_work()
2737 		mrioc->unrecoverable = 1;  in mpi3mr_watchdog_work()
2740 	if (mrioc->unrecoverable) {  in mpi3mr_watchdog_work()
2747 	if (mrioc->ts_update_counter++ >= mrioc->ts_update_interval) {  in mpi3mr_watchdog_work()
2748 		mrioc->ts_update_counter = 0;  in mpi3mr_watchdog_work()
2752 	if ((mrioc->prepare_for_reset) &&  in mpi3mr_watchdog_work()
2753 	    ((mrioc->prepare_for_reset_timeout_counter++) >=  in mpi3mr_watchdog_work()
2761 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_watchdog_work()
2774 	trigger_data.fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;  in mpi3mr_watchdog_work()
2777 	host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_watchdog_work()
2779 		if (!mrioc->diagsave_timeout) {  in mpi3mr_watchdog_work()
2783 		if ((mrioc->diagsave_timeout++) <= MPI3_SYSIF_DIAG_SAVE_TIMEOUT)  in mpi3mr_watchdog_work()
2788 	mrioc->diagsave_timeout = 0;  in mpi3mr_watchdog_work()
2791 		mrioc->unrecoverable = 1;  in mpi3mr_watchdog_work()
2800 		mrioc->unrecoverable = 1;  in mpi3mr_watchdog_work()
2814 	spin_lock_irqsave(&mrioc->watchdog_lock, flags);  in mpi3mr_watchdog_work()
2815 	if (mrioc->watchdog_work_q)  in mpi3mr_watchdog_work()
2816 		queue_delayed_work(mrioc->watchdog_work_q,  in mpi3mr_watchdog_work()
2817 		    &mrioc->watchdog_work,  in mpi3mr_watchdog_work()
2819 	spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);  in mpi3mr_watchdog_work()
2824  * mpi3mr_start_watchdog - Start watchdog
2834 	if (mrioc->watchdog_work_q)  in mpi3mr_start_watchdog()
2837 	INIT_DELAYED_WORK(&mrioc->watchdog_work, mpi3mr_watchdog_work);  in mpi3mr_start_watchdog()
2838 	snprintf(mrioc->watchdog_work_q_name,  in mpi3mr_start_watchdog()
2839 	    sizeof(mrioc->watchdog_work_q_name), "watchdog_%s%d", mrioc->name,  in mpi3mr_start_watchdog()
2840 	    mrioc->id);  in mpi3mr_start_watchdog()
2841 	mrioc->watchdog_work_q = alloc_ordered_workqueue(  in mpi3mr_start_watchdog()
2842 		"%s", WQ_MEM_RECLAIM, mrioc->watchdog_work_q_name);  in mpi3mr_start_watchdog()
2843 	if (!mrioc->watchdog_work_q) {  in mpi3mr_start_watchdog()
2848 	if (mrioc->watchdog_work_q)  in mpi3mr_start_watchdog()
2849 		queue_delayed_work(mrioc->watchdog_work_q,  in mpi3mr_start_watchdog()
2850 		    &mrioc->watchdog_work,  in mpi3mr_start_watchdog()
2855  * mpi3mr_stop_watchdog - Stop watchdog
2868 	spin_lock_irqsave(&mrioc->watchdog_lock, flags);  in mpi3mr_stop_watchdog()
2869 	wq = mrioc->watchdog_work_q;  in mpi3mr_stop_watchdog()
2870 	mrioc->watchdog_work_q = NULL;  in mpi3mr_stop_watchdog()
2871 	spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);  in mpi3mr_stop_watchdog()
2873 		if (!cancel_delayed_work_sync(&mrioc->watchdog_work))  in mpi3mr_stop_watchdog()
2880  * mpi3mr_setup_admin_qpair - Setup admin queue pair
2886  * Return: 0 on success, non-zero on failures.
2893 	mrioc->admin_req_q_sz = MPI3MR_ADMIN_REQ_Q_SIZE;  in mpi3mr_setup_admin_qpair()
2894 	mrioc->num_admin_req = mrioc->admin_req_q_sz /  in mpi3mr_setup_admin_qpair()
2896 	mrioc->admin_req_ci = mrioc->admin_req_pi = 0;  in mpi3mr_setup_admin_qpair()
2898 	mrioc->admin_reply_q_sz = MPI3MR_ADMIN_REPLY_Q_SIZE;  in mpi3mr_setup_admin_qpair()
2899 	mrioc->num_admin_replies = mrioc->admin_reply_q_sz /  in mpi3mr_setup_admin_qpair()
2901 	mrioc->admin_reply_ci = 0;  in mpi3mr_setup_admin_qpair()
2902 	mrioc->admin_reply_ephase = 1;  in mpi3mr_setup_admin_qpair()
2903 	atomic_set(&mrioc->admin_reply_q_in_use, 0);  in mpi3mr_setup_admin_qpair()
2905 	if (!mrioc->admin_req_base) {  in mpi3mr_setup_admin_qpair()
2906 		mrioc->admin_req_base = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_setup_admin_qpair()
2907 		    mrioc->admin_req_q_sz, &mrioc->admin_req_dma, GFP_KERNEL);  in mpi3mr_setup_admin_qpair()
2909 		if (!mrioc->admin_req_base) {  in mpi3mr_setup_admin_qpair()
2910 			retval = -1;  in mpi3mr_setup_admin_qpair()
2914 		mrioc->admin_reply_base = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_setup_admin_qpair()
2915 		    mrioc->admin_reply_q_sz, &mrioc->admin_reply_dma,  in mpi3mr_setup_admin_qpair()
2918 		if (!mrioc->admin_reply_base) {  in mpi3mr_setup_admin_qpair()
2919 			retval = -1;  in mpi3mr_setup_admin_qpair()
2924 	num_admin_entries = (mrioc->num_admin_replies << 16) |  in mpi3mr_setup_admin_qpair()
2925 	    (mrioc->num_admin_req);  in mpi3mr_setup_admin_qpair()
2926 	writel(num_admin_entries, &mrioc->sysif_regs->admin_queue_num_entries);  in mpi3mr_setup_admin_qpair()
2927 	mpi3mr_writeq(mrioc->admin_req_dma,  in mpi3mr_setup_admin_qpair()
2928 	    &mrioc->sysif_regs->admin_request_queue_address);  in mpi3mr_setup_admin_qpair()
2929 	mpi3mr_writeq(mrioc->admin_reply_dma,  in mpi3mr_setup_admin_qpair()
2930 	    &mrioc->sysif_regs->admin_reply_queue_address);  in mpi3mr_setup_admin_qpair()
2931 	writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);  in mpi3mr_setup_admin_qpair()
2932 	writel(mrioc->admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);  in mpi3mr_setup_admin_qpair()
2937 	if (mrioc->admin_reply_base) {  in mpi3mr_setup_admin_qpair()
2938 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,  in mpi3mr_setup_admin_qpair()
2939 		    mrioc->admin_reply_base, mrioc->admin_reply_dma);  in mpi3mr_setup_admin_qpair()
2940 		mrioc->admin_reply_base = NULL;  in mpi3mr_setup_admin_qpair()
2942 	if (mrioc->admin_req_base) {  in mpi3mr_setup_admin_qpair()
2943 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,  in mpi3mr_setup_admin_qpair()
2944 		    mrioc->admin_req_base, mrioc->admin_req_dma);  in mpi3mr_setup_admin_qpair()
2945 		mrioc->admin_req_base = NULL;  in mpi3mr_setup_admin_qpair()
2951  * mpi3mr_issue_iocfacts - Send IOC Facts
2958  * Return: 0 on success, non-zero on failures.
2970 	data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,  in mpi3mr_issue_iocfacts()
2974 		retval = -1;  in mpi3mr_issue_iocfacts()
2979 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocfacts()
2980 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_iocfacts()
2981 		retval = -1;  in mpi3mr_issue_iocfacts()
2983 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocfacts()
2986 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_iocfacts()
2987 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_iocfacts()
2988 	mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_iocfacts()
2995 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_iocfacts()
3002 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_issue_iocfacts()
3004 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_iocfacts()
3008 		retval = -1;  in mpi3mr_issue_iocfacts()
3011 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_iocfacts()
3015 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_iocfacts()
3016 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_issue_iocfacts()
3017 		retval = -1;  in mpi3mr_issue_iocfacts()
3023 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_iocfacts()
3024 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocfacts()
3028 		dma_free_coherent(&mrioc->pdev->dev, data_len, data, data_dma);  in mpi3mr_issue_iocfacts()
3034  * mpi3mr_check_reset_dma_mask - Process IOC facts data
3040  * Return: 0 on success, non-zero on failure.
3044 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_check_reset_dma_mask()
3046 	u64 facts_dma_mask = DMA_BIT_MASK(mrioc->facts.dma_mask);  in mpi3mr_check_reset_dma_mask()
3048 	if (!mrioc->facts.dma_mask || (mrioc->dma_mask <= facts_dma_mask))  in mpi3mr_check_reset_dma_mask()
3052 	    mrioc->dma_mask, facts_dma_mask);  in mpi3mr_check_reset_dma_mask()
3054 	r = dma_set_mask_and_coherent(&pdev->dev, facts_dma_mask);  in mpi3mr_check_reset_dma_mask()
3060 	mrioc->dma_mask = facts_dma_mask;  in mpi3mr_check_reset_dma_mask()
3065  * mpi3mr_process_factsdata - Process IOC facts data
3079 	if ((le16_to_cpu(facts_data->ioc_facts_data_length)) !=  in mpi3mr_process_factsdata()
3084 		    le16_to_cpu(facts_data->ioc_facts_data_length) * 4);  in mpi3mr_process_factsdata()
3087 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_process_factsdata()
3090 	if (le16_to_cpu(facts_data->ioc_request_frame_size) != (req_sz / 4)) {  in mpi3mr_process_factsdata()
3093 		    req_sz / 4, le16_to_cpu(facts_data->ioc_request_frame_size));  in mpi3mr_process_factsdata()
3096 	memset(&mrioc->facts, 0, sizeof(mrioc->facts));  in mpi3mr_process_factsdata()
3098 	facts_flags = le32_to_cpu(facts_data->flags);  in mpi3mr_process_factsdata()
3099 	mrioc->facts.op_req_sz = req_sz;  in mpi3mr_process_factsdata()
3100 	mrioc->op_reply_desc_sz = 1 << ((ioc_config &  in mpi3mr_process_factsdata()
3104 	mrioc->facts.ioc_num = facts_data->ioc_number;  in mpi3mr_process_factsdata()
3105 	mrioc->facts.who_init = facts_data->who_init;  in mpi3mr_process_factsdata()
3106 	mrioc->facts.max_msix_vectors = le16_to_cpu(facts_data->max_msix_vectors);  in mpi3mr_process_factsdata()
3107 	mrioc->facts.personality = (facts_flags &  in mpi3mr_process_factsdata()
3109 	mrioc->facts.dma_mask = (facts_flags &  in mpi3mr_process_factsdata()
3112 	mrioc->facts.dma_mask = (facts_flags &  in mpi3mr_process_factsdata()
3115 	mrioc->facts.protocol_flags = facts_data->protocol_flags;  in mpi3mr_process_factsdata()
3116 	mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word);  in mpi3mr_process_factsdata()
3117 	mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_requests);  in mpi3mr_process_factsdata()
3118 	mrioc->facts.product_id = le16_to_cpu(facts_data->product_id);  in mpi3mr_process_factsdata()
3119 	mrioc->facts.reply_sz = le16_to_cpu(facts_data->reply_frame_size) * 4;  in mpi3mr_process_factsdata()
3120 	mrioc->facts.exceptions = le16_to_cpu(facts_data->ioc_exceptions);  in mpi3mr_process_factsdata()
3121 	mrioc->facts.max_perids = le16_to_cpu(facts_data->max_persistent_id);  in mpi3mr_process_factsdata()
3122 	mrioc->facts.max_vds = le16_to_cpu(facts_data->max_vds);  in mpi3mr_process_factsdata()
3123 	mrioc->facts.max_hpds = le16_to_cpu(facts_data->max_host_pds);  in mpi3mr_process_factsdata()
3124 	mrioc->facts.max_advhpds = le16_to_cpu(facts_data->max_adv_host_pds);  in mpi3mr_process_factsdata()
3125 	mrioc->facts.max_raid_pds = le16_to_cpu(facts_data->max_raid_pds);  in mpi3mr_process_factsdata()
3126 	mrioc->facts.max_nvme = le16_to_cpu(facts_data->max_nvme);  in mpi3mr_process_factsdata()
3127 	mrioc->facts.max_pcie_switches =  in mpi3mr_process_factsdata()
3128 	    le16_to_cpu(facts_data->max_pcie_switches);  in mpi3mr_process_factsdata()
3129 	mrioc->facts.max_sasexpanders =  in mpi3mr_process_factsdata()
3130 	    le16_to_cpu(facts_data->max_sas_expanders);  in mpi3mr_process_factsdata()
3131 	mrioc->facts.max_data_length = le16_to_cpu(facts_data->max_data_length);  in mpi3mr_process_factsdata()
3132 	mrioc->facts.max_sasinitiators =  in mpi3mr_process_factsdata()
3133 	    le16_to_cpu(facts_data->max_sas_initiators);  in mpi3mr_process_factsdata()
3134 	mrioc->facts.max_enclosures = le16_to_cpu(facts_data->max_enclosures);  in mpi3mr_process_factsdata()
3135 	mrioc->facts.min_devhandle = le16_to_cpu(facts_data->min_dev_handle);  in mpi3mr_process_factsdata()
3136 	mrioc->facts.max_devhandle = le16_to_cpu(facts_data->max_dev_handle);  in mpi3mr_process_factsdata()
3137 	mrioc->facts.max_op_req_q =  in mpi3mr_process_factsdata()
3138 	    le16_to_cpu(facts_data->max_operational_request_queues);  in mpi3mr_process_factsdata()
3139 	mrioc->facts.max_op_reply_q =  in mpi3mr_process_factsdata()
3140 	    le16_to_cpu(facts_data->max_operational_reply_queues);  in mpi3mr_process_factsdata()
3141 	mrioc->facts.ioc_capabilities =  in mpi3mr_process_factsdata()
3142 	    le32_to_cpu(facts_data->ioc_capabilities);  in mpi3mr_process_factsdata()
3143 	mrioc->facts.fw_ver.build_num =  in mpi3mr_process_factsdata()
3144 	    le16_to_cpu(facts_data->fw_version.build_num);  in mpi3mr_process_factsdata()
3145 	mrioc->facts.fw_ver.cust_id =  in mpi3mr_process_factsdata()
3146 	    le16_to_cpu(facts_data->fw_version.customer_id);  in mpi3mr_process_factsdata()
3147 	mrioc->facts.fw_ver.ph_minor = facts_data->fw_version.phase_minor;  in mpi3mr_process_factsdata()
3148 	mrioc->facts.fw_ver.ph_major = facts_data->fw_version.phase_major;  in mpi3mr_process_factsdata()
3149 	mrioc->facts.fw_ver.gen_minor = facts_data->fw_version.gen_minor;  in mpi3mr_process_factsdata()
3150 	mrioc->facts.fw_ver.gen_major = facts_data->fw_version.gen_major;  in mpi3mr_process_factsdata()
3151 	mrioc->msix_count = min_t(int, mrioc->msix_count,  in mpi3mr_process_factsdata()
3152 	    mrioc->facts.max_msix_vectors);  in mpi3mr_process_factsdata()
3153 	mrioc->facts.sge_mod_mask = facts_data->sge_modifier_mask;  in mpi3mr_process_factsdata()
3154 	mrioc->facts.sge_mod_value = facts_data->sge_modifier_value;  in mpi3mr_process_factsdata()
3155 	mrioc->facts.sge_mod_shift = facts_data->sge_modifier_shift;  in mpi3mr_process_factsdata()
3156 	mrioc->facts.shutdown_timeout =  in mpi3mr_process_factsdata()
3157 	    le16_to_cpu(facts_data->shutdown_timeout);  in mpi3mr_process_factsdata()
3158 	mrioc->facts.diag_trace_sz =  in mpi3mr_process_factsdata()
3159 	    le32_to_cpu(facts_data->diag_trace_size);  in mpi3mr_process_factsdata()
3160 	mrioc->facts.diag_fw_sz =  in mpi3mr_process_factsdata()
3161 	    le32_to_cpu(facts_data->diag_fw_size);  in mpi3mr_process_factsdata()
3162 	mrioc->facts.diag_drvr_sz = le32_to_cpu(facts_data->diag_driver_size);  in mpi3mr_process_factsdata()
3163 	mrioc->facts.max_dev_per_tg =  in mpi3mr_process_factsdata()
3164 	    facts_data->max_devices_per_throttle_group;  in mpi3mr_process_factsdata()
3165 	mrioc->facts.io_throttle_data_length =  in mpi3mr_process_factsdata()
3166 	    le16_to_cpu(facts_data->io_throttle_data_length);  in mpi3mr_process_factsdata()
3167 	mrioc->facts.max_io_throttle_group =  in mpi3mr_process_factsdata()
3168 	    le16_to_cpu(facts_data->max_io_throttle_group);  in mpi3mr_process_factsdata()
3169 	mrioc->facts.io_throttle_low = le16_to_cpu(facts_data->io_throttle_low);  in mpi3mr_process_factsdata()
3170 	mrioc->facts.io_throttle_high =  in mpi3mr_process_factsdata()
3171 	    le16_to_cpu(facts_data->io_throttle_high);  in mpi3mr_process_factsdata()
3173 	if (mrioc->facts.max_data_length ==  in mpi3mr_process_factsdata()
3175 		mrioc->facts.max_data_length = MPI3MR_DEFAULT_MAX_IO_SIZE;  in mpi3mr_process_factsdata()
3177 		mrioc->facts.max_data_length *= MPI3MR_PAGE_SIZE_4K;  in mpi3mr_process_factsdata()
3179 	if (mrioc->facts.io_throttle_data_length)  in mpi3mr_process_factsdata()
3180 		mrioc->io_throttle_data_length =  in mpi3mr_process_factsdata()
3181 		    (mrioc->facts.io_throttle_data_length * 2 * 4);  in mpi3mr_process_factsdata()
3184 		mrioc->io_throttle_data_length = (mrioc->facts.max_data_length / 512) + 2;  in mpi3mr_process_factsdata()
3186 	mrioc->io_throttle_high = (mrioc->facts.io_throttle_high * 2 * 1024);  in mpi3mr_process_factsdata()
3187 	mrioc->io_throttle_low = (mrioc->facts.io_throttle_low * 2 * 1024);  in mpi3mr_process_factsdata()
3190 	    mrioc->facts.ioc_num, mrioc->facts.max_op_req_q,  in mpi3mr_process_factsdata()
3191 	    mrioc->facts.max_op_reply_q, mrioc->facts.max_devhandle);  in mpi3mr_process_factsdata()
3194 	    mrioc->facts.max_reqs, mrioc->facts.min_devhandle,  in mpi3mr_process_factsdata()
3195 	    mrioc->facts.max_msix_vectors, mrioc->facts.max_perids);  in mpi3mr_process_factsdata()
3197 	    mrioc->facts.sge_mod_mask, mrioc->facts.sge_mod_value,  in mpi3mr_process_factsdata()
3198 	    mrioc->facts.sge_mod_shift);  in mpi3mr_process_factsdata()
3200 	    mrioc->facts.dma_mask, (facts_flags &  in mpi3mr_process_factsdata()
3201 	    MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK), mrioc->facts.max_data_length);  in mpi3mr_process_factsdata()
3204 	    mrioc->facts.max_dev_per_tg, mrioc->facts.max_io_throttle_group);  in mpi3mr_process_factsdata()
3207 	   mrioc->facts.io_throttle_data_length * 4,  in mpi3mr_process_factsdata()
3208 	   mrioc->facts.io_throttle_high, mrioc->facts.io_throttle_low);  in mpi3mr_process_factsdata()
3212  * mpi3mr_alloc_reply_sense_bufs - Send IOC Init
3218  * Return: 0 on success, non-zero on failures.
3225 	if (mrioc->init_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3228 	mrioc->init_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3229 	if (!mrioc->init_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3232 	mrioc->bsg_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3233 	if (!mrioc->bsg_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3236 	mrioc->transport_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3237 	if (!mrioc->transport_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3241 		mrioc->dev_rmhs_cmds[i].reply = kzalloc(mrioc->reply_sz,  in mpi3mr_alloc_reply_sense_bufs()
3243 		if (!mrioc->dev_rmhs_cmds[i].reply)  in mpi3mr_alloc_reply_sense_bufs()
3248 		mrioc->evtack_cmds[i].reply = kzalloc(mrioc->reply_sz,  in mpi3mr_alloc_reply_sense_bufs()
3250 		if (!mrioc->evtack_cmds[i].reply)  in mpi3mr_alloc_reply_sense_bufs()
3254 	mrioc->host_tm_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3255 	if (!mrioc->host_tm_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3258 	mrioc->pel_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3259 	if (!mrioc->pel_cmds.reply)  in mpi3mr_alloc_reply_sense_bufs()
3262 	mrioc->pel_abort_cmd.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3263 	if (!mrioc->pel_abort_cmd.reply)  in mpi3mr_alloc_reply_sense_bufs()
3266 	mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;  in mpi3mr_alloc_reply_sense_bufs()
3267 	mrioc->removepend_bitmap = bitmap_zalloc(mrioc->dev_handle_bitmap_bits,  in mpi3mr_alloc_reply_sense_bufs()
3269 	if (!mrioc->removepend_bitmap)  in mpi3mr_alloc_reply_sense_bufs()
3272 	mrioc->devrem_bitmap = bitmap_zalloc(MPI3MR_NUM_DEVRMCMD, GFP_KERNEL);  in mpi3mr_alloc_reply_sense_bufs()
3273 	if (!mrioc->devrem_bitmap)  in mpi3mr_alloc_reply_sense_bufs()
3276 	mrioc->evtack_cmds_bitmap = bitmap_zalloc(MPI3MR_NUM_EVTACKCMD,  in mpi3mr_alloc_reply_sense_bufs()
3278 	if (!mrioc->evtack_cmds_bitmap)  in mpi3mr_alloc_reply_sense_bufs()
3281 	mrioc->num_reply_bufs = mrioc->facts.max_reqs + MPI3MR_NUM_EVT_REPLIES;  in mpi3mr_alloc_reply_sense_bufs()
3282 	mrioc->reply_free_qsz = mrioc->num_reply_bufs + 1;  in mpi3mr_alloc_reply_sense_bufs()
3283 	mrioc->num_sense_bufs = mrioc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;  in mpi3mr_alloc_reply_sense_bufs()
3284 	mrioc->sense_buf_q_sz = mrioc->num_sense_bufs + 1;  in mpi3mr_alloc_reply_sense_bufs()
3287 	sz = mrioc->num_reply_bufs * mrioc->reply_sz;  in mpi3mr_alloc_reply_sense_bufs()
3288 	mrioc->reply_buf_pool = dma_pool_create("reply_buf pool",  in mpi3mr_alloc_reply_sense_bufs()
3289 	    &mrioc->pdev->dev, sz, 16, 0);  in mpi3mr_alloc_reply_sense_bufs()
3290 	if (!mrioc->reply_buf_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3295 	mrioc->reply_buf = dma_pool_zalloc(mrioc->reply_buf_pool, GFP_KERNEL,  in mpi3mr_alloc_reply_sense_bufs()
3296 	    &mrioc->reply_buf_dma);  in mpi3mr_alloc_reply_sense_bufs()
3297 	if (!mrioc->reply_buf)  in mpi3mr_alloc_reply_sense_bufs()
3300 	mrioc->reply_buf_dma_max_address = mrioc->reply_buf_dma + sz;  in mpi3mr_alloc_reply_sense_bufs()
3303 	sz = mrioc->reply_free_qsz * 8;  in mpi3mr_alloc_reply_sense_bufs()
3304 	mrioc->reply_free_q_pool = dma_pool_create("reply_free_q pool",  in mpi3mr_alloc_reply_sense_bufs()
3305 	    &mrioc->pdev->dev, sz, 8, 0);  in mpi3mr_alloc_reply_sense_bufs()
3306 	if (!mrioc->reply_free_q_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3310 	mrioc->reply_free_q = dma_pool_zalloc(mrioc->reply_free_q_pool,  in mpi3mr_alloc_reply_sense_bufs()
3311 	    GFP_KERNEL, &mrioc->reply_free_q_dma);  in mpi3mr_alloc_reply_sense_bufs()
3312 	if (!mrioc->reply_free_q)  in mpi3mr_alloc_reply_sense_bufs()
3316 	sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;  in mpi3mr_alloc_reply_sense_bufs()
3317 	mrioc->sense_buf_pool = dma_pool_create("sense_buf pool",  in mpi3mr_alloc_reply_sense_bufs()
3318 	    &mrioc->pdev->dev, sz, 4, 0);  in mpi3mr_alloc_reply_sense_bufs()
3319 	if (!mrioc->sense_buf_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3323 	mrioc->sense_buf = dma_pool_zalloc(mrioc->sense_buf_pool, GFP_KERNEL,  in mpi3mr_alloc_reply_sense_bufs()
3324 	    &mrioc->sense_buf_dma);  in mpi3mr_alloc_reply_sense_bufs()
3325 	if (!mrioc->sense_buf)  in mpi3mr_alloc_reply_sense_bufs()
3329 	sz = mrioc->sense_buf_q_sz * 8;  in mpi3mr_alloc_reply_sense_bufs()
3330 	mrioc->sense_buf_q_pool = dma_pool_create("sense_buf_q pool",  in mpi3mr_alloc_reply_sense_bufs()
3331 	    &mrioc->pdev->dev, sz, 8, 0);  in mpi3mr_alloc_reply_sense_bufs()
3332 	if (!mrioc->sense_buf_q_pool) {  in mpi3mr_alloc_reply_sense_bufs()
3336 	mrioc->sense_buf_q = dma_pool_zalloc(mrioc->sense_buf_q_pool,  in mpi3mr_alloc_reply_sense_bufs()
3337 	    GFP_KERNEL, &mrioc->sense_buf_q_dma);  in mpi3mr_alloc_reply_sense_bufs()
3338 	if (!mrioc->sense_buf_q)  in mpi3mr_alloc_reply_sense_bufs()
3344 	retval = -1;  in mpi3mr_alloc_reply_sense_bufs()
3349  * mpimr_initialize_reply_sbuf_queues - initialize reply sense
3363 	sz = mrioc->num_reply_bufs * mrioc->reply_sz;  in mpimr_initialize_reply_sbuf_queues()
3366 	    mrioc->reply_buf, mrioc->num_reply_bufs, mrioc->reply_sz,  in mpimr_initialize_reply_sbuf_queues()
3367 	    (sz / 1024), (unsigned long long)mrioc->reply_buf_dma);  in mpimr_initialize_reply_sbuf_queues()
3368 	sz = mrioc->reply_free_qsz * 8;  in mpimr_initialize_reply_sbuf_queues()
3371 	    mrioc->reply_free_q, mrioc->reply_free_qsz, 8, (sz / 1024),  in mpimr_initialize_reply_sbuf_queues()
3372 	    (unsigned long long)mrioc->reply_free_q_dma);  in mpimr_initialize_reply_sbuf_queues()
3373 	sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;  in mpimr_initialize_reply_sbuf_queues()
3376 	    mrioc->sense_buf, mrioc->num_sense_bufs, MPI3MR_SENSE_BUF_SZ,  in mpimr_initialize_reply_sbuf_queues()
3377 	    (sz / 1024), (unsigned long long)mrioc->sense_buf_dma);  in mpimr_initialize_reply_sbuf_queues()
3378 	sz = mrioc->sense_buf_q_sz * 8;  in mpimr_initialize_reply_sbuf_queues()
3381 	    mrioc->sense_buf_q, mrioc->sense_buf_q_sz, 8, (sz / 1024),  in mpimr_initialize_reply_sbuf_queues()
3382 	    (unsigned long long)mrioc->sense_buf_q_dma);  in mpimr_initialize_reply_sbuf_queues()
3385 	for (i = 0, phy_addr = mrioc->reply_buf_dma;  in mpimr_initialize_reply_sbuf_queues()
3386 	    i < mrioc->num_reply_bufs; i++, phy_addr += mrioc->reply_sz)  in mpimr_initialize_reply_sbuf_queues()
3387 		mrioc->reply_free_q[i] = cpu_to_le64(phy_addr);  in mpimr_initialize_reply_sbuf_queues()
3388 	mrioc->reply_free_q[i] = cpu_to_le64(0);  in mpimr_initialize_reply_sbuf_queues()
3391 	for (i = 0, phy_addr = mrioc->sense_buf_dma;  in mpimr_initialize_reply_sbuf_queues()
3392 	    i < mrioc->num_sense_bufs; i++, phy_addr += MPI3MR_SENSE_BUF_SZ)  in mpimr_initialize_reply_sbuf_queues()
3393 		mrioc->sense_buf_q[i] = cpu_to_le64(phy_addr);  in mpimr_initialize_reply_sbuf_queues()
3394 	mrioc->sense_buf_q[i] = cpu_to_le64(0);  in mpimr_initialize_reply_sbuf_queues()
3398  * mpi3mr_issue_iocinit - Send IOC Init
3404  * Return: 0 on success, non-zero on failures.
3415 	drv_info = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,  in mpi3mr_issue_iocinit()
3418 		retval = -1;  in mpi3mr_issue_iocinit()
3423 	drv_info->information_length = cpu_to_le32(data_len);  in mpi3mr_issue_iocinit()
3424 	strscpy(drv_info->driver_signature, "Broadcom", sizeof(drv_info->driver_signature));  in mpi3mr_issue_iocinit()
3425 	strscpy(drv_info->os_name, utsname()->sysname, sizeof(drv_info->os_name));  in mpi3mr_issue_iocinit()
3426 	strscpy(drv_info->os_version, utsname()->release, sizeof(drv_info->os_version));  in mpi3mr_issue_iocinit()
3427 	strscpy(drv_info->driver_name, MPI3MR_DRIVER_NAME, sizeof(drv_info->driver_name));  in mpi3mr_issue_iocinit()
3428 	strscpy(drv_info->driver_version, MPI3MR_DRIVER_VERSION, sizeof(drv_info->driver_version));  in mpi3mr_issue_iocinit()
3429 	strscpy(drv_info->driver_release_date, MPI3MR_DRIVER_RELDATE,  in mpi3mr_issue_iocinit()
3430 	    sizeof(drv_info->driver_release_date));  in mpi3mr_issue_iocinit()
3431 	drv_info->driver_capabilities = 0;  in mpi3mr_issue_iocinit()
3432 	memcpy((u8 *)&mrioc->driver_info, (u8 *)drv_info,  in mpi3mr_issue_iocinit()
3433 	    sizeof(mrioc->driver_info));  in mpi3mr_issue_iocinit()
3436 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocinit()
3437 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_iocinit()
3438 		retval = -1;  in mpi3mr_issue_iocinit()
3440 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocinit()
3443 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_iocinit()
3444 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_iocinit()
3445 	mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_iocinit()
3453 	iocinit_req.reply_free_queue_depth = cpu_to_le16(mrioc->reply_free_qsz);  in mpi3mr_issue_iocinit()
3455 	    cpu_to_le64(mrioc->reply_free_q_dma);  in mpi3mr_issue_iocinit()
3458 	    cpu_to_le16(mrioc->sense_buf_q_sz);  in mpi3mr_issue_iocinit()
3460 	    cpu_to_le64(mrioc->sense_buf_q_dma);  in mpi3mr_issue_iocinit()
3471 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_iocinit()
3478 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_issue_iocinit()
3480 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_iocinit()
3484 		retval = -1;  in mpi3mr_issue_iocinit()
3487 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_iocinit()
3491 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_iocinit()
3492 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_issue_iocinit()
3493 		retval = -1;  in mpi3mr_issue_iocinit()
3497 	mrioc->reply_free_queue_host_index = mrioc->num_reply_bufs;  in mpi3mr_issue_iocinit()
3498 	writel(mrioc->reply_free_queue_host_index,  in mpi3mr_issue_iocinit()
3499 	    &mrioc->sysif_regs->reply_free_host_index);  in mpi3mr_issue_iocinit()
3501 	mrioc->sbq_host_index = mrioc->num_sense_bufs;  in mpi3mr_issue_iocinit()
3502 	writel(mrioc->sbq_host_index,  in mpi3mr_issue_iocinit()
3503 	    &mrioc->sysif_regs->sense_buffer_free_host_index);  in mpi3mr_issue_iocinit()
3505 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_iocinit()
3506 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_iocinit()
3510 		dma_free_coherent(&mrioc->pdev->dev, data_len, drv_info,  in mpi3mr_issue_iocinit()
3517  * mpi3mr_unmask_events - Unmask events in event mask bitmap
3524  * Return: 0 on success, non-zero on failures.
3537 	mrioc->event_masks[word] &= ~desired_event;  in mpi3mr_unmask_events()
3541  * mpi3mr_issue_event_notification - Send event notification
3547  * Return: 0 on success, non-zero on failures.
3556 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_event_notification()
3557 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_event_notification()
3558 		retval = -1;  in mpi3mr_issue_event_notification()
3560 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_event_notification()
3563 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_event_notification()
3564 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_event_notification()
3565 	mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_event_notification()
3570 		    cpu_to_le32(mrioc->event_masks[i]);  in mpi3mr_issue_event_notification()
3571 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_event_notification()
3578 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_issue_event_notification()
3580 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_event_notification()
3584 		retval = -1;  in mpi3mr_issue_event_notification()
3587 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_issue_event_notification()
3591 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_issue_event_notification()
3592 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_issue_event_notification()
3593 		retval = -1;  in mpi3mr_issue_event_notification()
3598 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_event_notification()
3599 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_event_notification()
3605  * mpi3mr_process_event_ack - Process event acknowledgment
3613  * Return: 0 on success, non-zero on failures.
3622 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_process_event_ack()
3623 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_process_event_ack()
3624 		retval = -1;  in mpi3mr_process_event_ack()
3626 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_process_event_ack()
3629 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_process_event_ack()
3630 	mrioc->init_cmds.is_waiting = 1;  in mpi3mr_process_event_ack()
3631 	mrioc->init_cmds.callback = NULL;  in mpi3mr_process_event_ack()
3637 	init_completion(&mrioc->init_cmds.done);  in mpi3mr_process_event_ack()
3644 	wait_for_completion_timeout(&mrioc->init_cmds.done,  in mpi3mr_process_event_ack()
3646 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_process_event_ack()
3648 		if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))  in mpi3mr_process_event_ack()
3651 		retval = -1;  in mpi3mr_process_event_ack()
3654 	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)  in mpi3mr_process_event_ack()
3658 		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),  in mpi3mr_process_event_ack()
3659 		    mrioc->init_cmds.ioc_loginfo);  in mpi3mr_process_event_ack()
3660 		retval = -1;  in mpi3mr_process_event_ack()
3665 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_process_event_ack()
3666 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_process_event_ack()
3672  * mpi3mr_alloc_chain_bufs - Allocate chain buffers
3679  * Return: 0 on success, non-zero on failure
3687 	if (mrioc->chain_sgl_list)  in mpi3mr_alloc_chain_bufs()
3690 	num_chains = mrioc->max_host_ios / MPI3MR_CHAINBUF_FACTOR;  in mpi3mr_alloc_chain_bufs()
3698 	mrioc->chain_buf_count = num_chains;  in mpi3mr_alloc_chain_bufs()
3700 	mrioc->chain_sgl_list = kzalloc(sz, GFP_KERNEL);  in mpi3mr_alloc_chain_bufs()
3701 	if (!mrioc->chain_sgl_list)  in mpi3mr_alloc_chain_bufs()
3704 	if (mrioc->max_sgl_entries > (mrioc->facts.max_data_length /  in mpi3mr_alloc_chain_bufs()
3706 		mrioc->max_sgl_entries = mrioc->facts.max_data_length /  in mpi3mr_alloc_chain_bufs()
3708 	sz = mrioc->max_sgl_entries * sizeof(struct mpi3_sge_common);  in mpi3mr_alloc_chain_bufs()
3710 			mrioc->max_sgl_entries, sz/1024);  in mpi3mr_alloc_chain_bufs()
3712 	mrioc->chain_buf_pool = dma_pool_create("chain_buf pool",  in mpi3mr_alloc_chain_bufs()
3713 	    &mrioc->pdev->dev, sz, 16, 0);  in mpi3mr_alloc_chain_bufs()
3714 	if (!mrioc->chain_buf_pool) {  in mpi3mr_alloc_chain_bufs()
3720 		mrioc->chain_sgl_list[i].addr =  in mpi3mr_alloc_chain_bufs()
3721 		    dma_pool_zalloc(mrioc->chain_buf_pool, GFP_KERNEL,  in mpi3mr_alloc_chain_bufs()
3722 		    &mrioc->chain_sgl_list[i].dma_addr);  in mpi3mr_alloc_chain_bufs()
3724 		if (!mrioc->chain_sgl_list[i].addr)  in mpi3mr_alloc_chain_bufs()
3727 	mrioc->chain_bitmap = bitmap_zalloc(num_chains, GFP_KERNEL);  in mpi3mr_alloc_chain_bufs()
3728 	if (!mrioc->chain_bitmap)  in mpi3mr_alloc_chain_bufs()
3732 	retval = -1;  in mpi3mr_alloc_chain_bufs()
3737  * mpi3mr_port_enable_complete - Mark port enable complete
3749 	drv_cmd->callback = NULL;  in mpi3mr_port_enable_complete()
3750 	mrioc->scan_started = 0;  in mpi3mr_port_enable_complete()
3751 	if (drv_cmd->state & MPI3MR_CMD_RESET)  in mpi3mr_port_enable_complete()
3752 		mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;  in mpi3mr_port_enable_complete()
3754 		mrioc->scan_failed = drv_cmd->ioc_status;  in mpi3mr_port_enable_complete()
3755 	drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_port_enable_complete()
3759  * mpi3mr_issue_port_enable - Issue Port Enable
3767  * Return: 0 on success, non-zero on failures.
3776 	mutex_lock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3777 	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_issue_port_enable()
3778 		retval = -1;  in mpi3mr_issue_port_enable()
3780 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3783 	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_issue_port_enable()
3785 		mrioc->init_cmds.is_waiting = 0;  in mpi3mr_issue_port_enable()
3786 		mrioc->init_cmds.callback = mpi3mr_port_enable_complete;  in mpi3mr_issue_port_enable()
3788 		mrioc->init_cmds.is_waiting = 1;  in mpi3mr_issue_port_enable()
3789 		mrioc->init_cmds.callback = NULL;  in mpi3mr_issue_port_enable()
3790 		init_completion(&mrioc->init_cmds.done);  in mpi3mr_issue_port_enable()
3801 		mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3805 	wait_for_completion_timeout(&mrioc->init_cmds.done, (pe_timeout * HZ));  in mpi3mr_issue_port_enable()
3806 	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_issue_port_enable()
3808 		retval = -1;  in mpi3mr_issue_port_enable()
3812 	mpi3mr_port_enable_complete(mrioc, &mrioc->init_cmds);  in mpi3mr_issue_port_enable()
3815 	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_issue_port_enable()
3816 	mutex_unlock(&mrioc->init_cmds.mutex);  in mpi3mr_issue_port_enable()
3841  * mpi3mr_repost_diag_bufs - repost host diag buffers
3847  * Return: 0 on success, non-zero on failures.
3862 		return -1;  in mpi3mr_repost_diag_bufs()
3868 	    trace_hdb->status != MPI3MR_HDB_BUFSTATUS_NOT_ALLOCATED &&  in mpi3mr_repost_diag_bufs()
3869 	    trace_hdb->trigger_type != MPI3MR_HDB_TRIGGER_TYPE_GLOBAL &&  in mpi3mr_repost_diag_bufs()
3870 	    trace_hdb->trigger_type != MPI3MR_HDB_TRIGGER_TYPE_ELEMENT)  in mpi3mr_repost_diag_bufs()
3875 	if (fw_hdb && fw_hdb->status != MPI3MR_HDB_BUFSTATUS_NOT_ALLOCATED &&  in mpi3mr_repost_diag_bufs()
3876 	    fw_hdb->trigger_type != MPI3MR_HDB_TRIGGER_TYPE_GLOBAL &&  in mpi3mr_repost_diag_bufs()
3877 	    fw_hdb->trigger_type != MPI3MR_HDB_TRIGGER_TYPE_ELEMENT)  in mpi3mr_repost_diag_bufs()
3881 		global_trigger = le64_to_cpu(mrioc->driver_pg2->global_trigger);  in mpi3mr_repost_diag_bufs()
3891 		prev_trigger_type = trace_hdb->trigger_type;  in mpi3mr_repost_diag_bufs()
3892 		memcpy(&prev_trigger_data, &trace_hdb->trigger_data,  in mpi3mr_repost_diag_bufs()
3893 		    sizeof(trace_hdb->trigger_data));  in mpi3mr_repost_diag_bufs()
3900 			trace_hdb->trigger_type = prev_trigger_type;  in mpi3mr_repost_diag_bufs()
3901 			memcpy(&trace_hdb->trigger_data, &prev_trigger_data,  in mpi3mr_repost_diag_bufs()
3904 			return -1;  in mpi3mr_repost_diag_bufs()
3909 		prev_trigger_type = fw_hdb->trigger_type;  in mpi3mr_repost_diag_bufs()
3910 		memcpy(&prev_trigger_data, &fw_hdb->trigger_data,  in mpi3mr_repost_diag_bufs()
3911 		    sizeof(fw_hdb->trigger_data));  in mpi3mr_repost_diag_bufs()
3918 			fw_hdb->trigger_type = prev_trigger_type;  in mpi3mr_repost_diag_bufs()
3919 			memcpy(&fw_hdb->trigger_data, &prev_trigger_data,  in mpi3mr_repost_diag_bufs()
3922 			return -1;  in mpi3mr_repost_diag_bufs()
3929  * mpi3mr_read_tsu_interval - Update time stamp interval
3944 	mrioc->ts_update_interval = MPI3MR_TSUPDATE_INTERVAL;  in mpi3mr_read_tsu_interval()
3948 		mrioc->ts_update_interval = (driver_pg1.time_stamp_update * 60);  in mpi3mr_read_tsu_interval()
3952  * mpi3mr_print_ioc_info - Display controller information
3967 	struct mpi3mr_compimg_ver *fwver = &mrioc->facts.fw_ver;  in mpi3mr_print_ioc_info()
3969 	switch (mrioc->facts.personality) {  in mpi3mr_print_ioc_info()
3984 	    fwver->gen_major, fwver->gen_minor, fwver->ph_major,  in mpi3mr_print_ioc_info()
3985 	    fwver->ph_minor, fwver->cust_id, fwver->build_num);  in mpi3mr_print_ioc_info()
3988 		if (mrioc->facts.protocol_flags &  in mpi3mr_print_ioc_info()
3991 				    sizeof(protocol) - bytes_written, "%s%s",  in mpi3mr_print_ioc_info()
3999 		if (mrioc->facts.protocol_flags &  in mpi3mr_print_ioc_info()
4002 				    sizeof(capabilities) - bytes_written, "%s%s",  in mpi3mr_print_ioc_info()
4013  * mpi3mr_cleanup_resources - Free PCI resources
4018  * Return: 0 on success and non-zero on failure.
4022 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_cleanup_resources()
4026 	if (mrioc->sysif_regs) {  in mpi3mr_cleanup_resources()
4027 		iounmap((void __iomem *)mrioc->sysif_regs);  in mpi3mr_cleanup_resources()
4028 		mrioc->sysif_regs = NULL;  in mpi3mr_cleanup_resources()
4032 		if (mrioc->bars)  in mpi3mr_cleanup_resources()
4033 			pci_release_selected_regions(pdev, mrioc->bars);  in mpi3mr_cleanup_resources()
4039  * mpi3mr_setup_resources - Enable PCI resources
4042  * Enable PCI device memory, MSI-x registers and set DMA mask.
4044  * Return: 0 on success and non-zero on failure.
4048 	struct pci_dev *pdev = mrioc->pdev;  in mpi3mr_setup_resources()
4052 	u64 dma_mask = mrioc->dma_mask ? mrioc->dma_mask :  in mpi3mr_setup_resources()
4057 		retval = -ENODEV;  in mpi3mr_setup_resources()
4063 		ioc_err(mrioc, "Unable to find MSI-X Capabilities\n");  in mpi3mr_setup_resources()
4064 		retval = -ENODEV;  in mpi3mr_setup_resources()
4067 	mrioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);  in mpi3mr_setup_resources()
4069 	if (pci_request_selected_regions(pdev, mrioc->bars,  in mpi3mr_setup_resources()
4070 	    mrioc->driver_name)) {  in mpi3mr_setup_resources()
4072 		retval = -ENODEV;  in mpi3mr_setup_resources()
4078 			mrioc->sysif_regs_phys = pci_resource_start(pdev, i);  in mpi3mr_setup_resources()
4080 			mrioc->sysif_regs =  in mpi3mr_setup_resources()
4081 			    ioremap(mrioc->sysif_regs_phys, memap_sz);  in mpi3mr_setup_resources()
4088 	retval = dma_set_mask_and_coherent(&pdev->dev, dma_mask);  in mpi3mr_setup_resources()
4093 			retval = dma_set_mask_and_coherent(&pdev->dev,  in mpi3mr_setup_resources()
4097 			mrioc->dma_mask = 0;  in mpi3mr_setup_resources()
4102 	mrioc->dma_mask = dma_mask;  in mpi3mr_setup_resources()
4104 	if (!mrioc->sysif_regs) {  in mpi3mr_setup_resources()
4107 		retval = -EINVAL;  in mpi3mr_setup_resources()
4112 	mrioc->msix_count = (message_control & 0x3FF) + 1;  in mpi3mr_setup_resources()
4116 	pci_set_drvdata(pdev, mrioc->shost);  in mpi3mr_setup_resources()
4121 	    (unsigned long long)mrioc->sysif_regs_phys,  in mpi3mr_setup_resources()
4122 	    mrioc->sysif_regs, memap_sz);  in mpi3mr_setup_resources()
4123 	ioc_info(mrioc, "Number of MSI-X vectors found in capabilities: (%d)\n",  in mpi3mr_setup_resources()
4124 	    mrioc->msix_count);  in mpi3mr_setup_resources()
4127 		mrioc->requested_poll_qcount = min_t(int, poll_queues,  in mpi3mr_setup_resources()
4128 				mrioc->msix_count - 2);  in mpi3mr_setup_resources()
4137  * mpi3mr_enable_events - Enable required events
4144  * Return: 0 on success and non-zero on failure.
4152 		mrioc->event_masks[i] = -1;  in mpi3mr_enable_events()
4178  * mpi3mr_init_ioc - Initialize the controller
4184  * registers, create admin and operational reply queue pairs,
4190  * Return: 0 on success and non-zero on failure.
4221 	mrioc->max_host_ios = mrioc->facts.max_reqs - MPI3MR_INTERNAL_CMDS_RESVD;  in mpi3mr_init_ioc()
4222 	mrioc->shost->max_sectors = mrioc->facts.max_data_length / 512;  in mpi3mr_init_ioc()
4223 	mrioc->num_io_throttle_group = mrioc->facts.max_io_throttle_group;  in mpi3mr_init_ioc()
4224 	atomic_set(&mrioc->pend_large_data_sz, 0);  in mpi3mr_init_ioc()
4227 		mrioc->max_host_ios = min_t(int, mrioc->max_host_ios,  in mpi3mr_init_ioc()
4230 	if (!(mrioc->facts.ioc_capabilities &  in mpi3mr_init_ioc()
4232 		mrioc->sas_transport_enabled = 1;  in mpi3mr_init_ioc()
4233 		mrioc->scsi_device_channel = 1;  in mpi3mr_init_ioc()
4234 		mrioc->shost->max_channel = 1;  in mpi3mr_init_ioc()
4235 		mrioc->shost->transportt = mpi3mr_transport_template;  in mpi3mr_init_ioc()
4238 	if (mrioc->facts.max_req_limit)  in mpi3mr_init_ioc()
4239 		mrioc->prevent_reply_qfull = true;  in mpi3mr_init_ioc()
4241 	mrioc->reply_sz = mrioc->facts.reply_sz;  in mpi3mr_init_ioc()
4265 	if (!mrioc->init_cmds.reply) {  in mpi3mr_init_ioc()
4275 	if (!mrioc->chain_sgl_list) {  in mpi3mr_init_ioc()
4299 		ioc_err(mrioc, "Failed to re-setup ISR, error %d\n",  in mpi3mr_init_ioc()
4311 	if (!mrioc->pel_seqnum_virt) {  in mpi3mr_init_ioc()
4313 		mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);  in mpi3mr_init_ioc()
4314 		mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_init_ioc()
4315 		    mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,  in mpi3mr_init_ioc()
4317 		if (!mrioc->pel_seqnum_virt) {  in mpi3mr_init_ioc()
4318 			retval = -ENOMEM;  in mpi3mr_init_ioc()
4323 	if (!mrioc->throttle_groups && mrioc->num_io_throttle_group) {  in mpi3mr_init_ioc()
4326 		mrioc->throttle_groups = kcalloc(mrioc->num_io_throttle_group, sz, GFP_KERNEL);  in mpi3mr_init_ioc()
4327 		if (!mrioc->throttle_groups) {  in mpi3mr_init_ioc()
4328 			retval = -1;  in mpi3mr_init_ioc()
4356 	retval = -1;  in mpi3mr_init_ioc()
4361 	mrioc->unrecoverable = 1;  in mpi3mr_init_ioc()
4366  * mpi3mr_reinit_ioc - Re-Initialize the controller
4370  * This the controller re-initialization routine, executed from
4372  * operational reply queue pairs, allocate required memory for
4377  * Return: 0 on success and non-zero on failure.
4397 	mrioc->io_admin_reset_sync = 0;  in mpi3mr_reinit_ioc()
4398 	if (is_resume || mrioc->block_on_pci_err) {  in mpi3mr_reinit_ioc()
4450 	if (is_resume || mrioc->block_on_pci_err) {  in mpi3mr_reinit_ioc()
4454 			ioc_err(mrioc, "failed to re-setup ISR\n");  in mpi3mr_reinit_ioc()
4459 	dprint_reset(mrioc, "creating operational queue pairs\n");  in mpi3mr_reinit_ioc()
4462 		ioc_err(mrioc, "failed to create operational queue pairs\n");  in mpi3mr_reinit_ioc()
4466 	if (!mrioc->pel_seqnum_virt) {  in mpi3mr_reinit_ioc()
4468 		mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);  in mpi3mr_reinit_ioc()
4469 		mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_reinit_ioc()
4470 		    mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,  in mpi3mr_reinit_ioc()
4472 		if (!mrioc->pel_seqnum_virt) {  in mpi3mr_reinit_ioc()
4473 			retval = -ENOMEM;  in mpi3mr_reinit_ioc()
4478 	if (mrioc->shost->nr_hw_queues > mrioc->num_op_reply_q) {  in mpi3mr_reinit_ioc()
4480 		    "cannot create minimum number of operational queues expected:%d created:%d\n",  in mpi3mr_reinit_ioc()
4481 		    mrioc->shost->nr_hw_queues, mrioc->num_op_reply_q);  in mpi3mr_reinit_ioc()
4482 		retval = -1;  in mpi3mr_reinit_ioc()
4493 	mrioc->device_refresh_on = 1;  in mpi3mr_reinit_ioc()
4504 		if (mrioc->init_cmds.state == MPI3MR_CMD_NOTUSED)  in mpi3mr_reinit_ioc()
4506 		if (!pci_device_is_present(mrioc->pdev))  in mpi3mr_reinit_ioc()
4507 			mrioc->unrecoverable = 1;  in mpi3mr_reinit_ioc()
4508 		if (mrioc->unrecoverable) {  in mpi3mr_reinit_ioc()
4509 			retval = -1;  in mpi3mr_reinit_ioc()
4512 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_reinit_ioc()
4516 			mrioc->init_cmds.is_waiting = 0;  in mpi3mr_reinit_ioc()
4517 			mrioc->init_cmds.callback = NULL;  in mpi3mr_reinit_ioc()
4518 			mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_reinit_ioc()
4521 	} while (--pe_timeout);  in mpi3mr_reinit_ioc()
4527 		mrioc->init_cmds.is_waiting = 0;  in mpi3mr_reinit_ioc()
4528 		mrioc->init_cmds.callback = NULL;  in mpi3mr_reinit_ioc()
4529 		mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_reinit_ioc()
4531 	} else if (mrioc->scan_failed) {  in mpi3mr_reinit_ioc()
4534 		    mrioc->scan_failed);  in mpi3mr_reinit_ioc()
4539 	    (is_resume)?"resume":"re-initialization");  in mpi3mr_reinit_ioc()
4545 		    (is_resume)?"resume":"re-initialization", retry);  in mpi3mr_reinit_ioc()
4549 	retval = -1;  in mpi3mr_reinit_ioc()
4552 	    (is_resume)?"resume":"re-initialization");  in mpi3mr_reinit_ioc()
4555 	mrioc->unrecoverable = 1;  in mpi3mr_reinit_ioc()
4560  * mpi3mr_memset_op_reply_q_buffers - memset the operational reply queue's
4563  * @qidx: Operational reply queue index
4569 	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;  in mpi3mr_memset_op_reply_q_buffers()
4573 	if (!op_reply_q->q_segments)  in mpi3mr_memset_op_reply_q_buffers()
4576 	size = op_reply_q->segment_qd * mrioc->op_reply_desc_sz;  in mpi3mr_memset_op_reply_q_buffers()
4577 	segments = op_reply_q->q_segments;  in mpi3mr_memset_op_reply_q_buffers()
4578 	for (i = 0; i < op_reply_q->num_segments; i++)  in mpi3mr_memset_op_reply_q_buffers()
4583  * mpi3mr_memset_op_req_q_buffers - memset the operational request queue's
4586  * @qidx: Operational request queue index
4592 	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;  in mpi3mr_memset_op_req_q_buffers()
4596 	if (!op_req_q->q_segments)  in mpi3mr_memset_op_req_q_buffers()
4599 	size = op_req_q->segment_qd * mrioc->facts.op_req_sz;  in mpi3mr_memset_op_req_q_buffers()
4600 	segments = op_req_q->q_segments;  in mpi3mr_memset_op_req_q_buffers()
4601 	for (i = 0; i < op_req_q->num_segments; i++)  in mpi3mr_memset_op_req_q_buffers()
4606  * mpi3mr_memset_buffers - memset memory for a controller
4620 	mrioc->change_count = 0;  in mpi3mr_memset_buffers()
4621 	mrioc->active_poll_qcount = 0;  in mpi3mr_memset_buffers()
4622 	mrioc->default_qcount = 0;  in mpi3mr_memset_buffers()
4623 	if (mrioc->admin_req_base)  in mpi3mr_memset_buffers()
4624 		memset(mrioc->admin_req_base, 0, mrioc->admin_req_q_sz);  in mpi3mr_memset_buffers()
4625 	if (mrioc->admin_reply_base)  in mpi3mr_memset_buffers()
4626 		memset(mrioc->admin_reply_base, 0, mrioc->admin_reply_q_sz);  in mpi3mr_memset_buffers()
4627 	atomic_set(&mrioc->admin_reply_q_in_use, 0);  in mpi3mr_memset_buffers()
4629 	if (mrioc->init_cmds.reply) {  in mpi3mr_memset_buffers()
4630 		memset(mrioc->init_cmds.reply, 0, sizeof(*mrioc->init_cmds.reply));  in mpi3mr_memset_buffers()
4631 		memset(mrioc->bsg_cmds.reply, 0,  in mpi3mr_memset_buffers()
4632 		    sizeof(*mrioc->bsg_cmds.reply));  in mpi3mr_memset_buffers()
4633 		memset(mrioc->host_tm_cmds.reply, 0,  in mpi3mr_memset_buffers()
4634 		    sizeof(*mrioc->host_tm_cmds.reply));  in mpi3mr_memset_buffers()
4635 		memset(mrioc->pel_cmds.reply, 0,  in mpi3mr_memset_buffers()
4636 		    sizeof(*mrioc->pel_cmds.reply));  in mpi3mr_memset_buffers()
4637 		memset(mrioc->pel_abort_cmd.reply, 0,  in mpi3mr_memset_buffers()
4638 		    sizeof(*mrioc->pel_abort_cmd.reply));  in mpi3mr_memset_buffers()
4639 		memset(mrioc->transport_cmds.reply, 0,  in mpi3mr_memset_buffers()
4640 		    sizeof(*mrioc->transport_cmds.reply));  in mpi3mr_memset_buffers()
4642 			memset(mrioc->dev_rmhs_cmds[i].reply, 0,  in mpi3mr_memset_buffers()
4643 			    sizeof(*mrioc->dev_rmhs_cmds[i].reply));  in mpi3mr_memset_buffers()
4645 			memset(mrioc->evtack_cmds[i].reply, 0,  in mpi3mr_memset_buffers()
4646 			    sizeof(*mrioc->evtack_cmds[i].reply));  in mpi3mr_memset_buffers()
4647 		bitmap_clear(mrioc->removepend_bitmap, 0,  in mpi3mr_memset_buffers()
4648 			     mrioc->dev_handle_bitmap_bits);  in mpi3mr_memset_buffers()
4649 		bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);  in mpi3mr_memset_buffers()
4650 		bitmap_clear(mrioc->evtack_cmds_bitmap, 0,  in mpi3mr_memset_buffers()
4654 	for (i = 0; i < mrioc->num_queues; i++) {  in mpi3mr_memset_buffers()
4655 		mrioc->op_reply_qinfo[i].qid = 0;  in mpi3mr_memset_buffers()
4656 		mrioc->op_reply_qinfo[i].ci = 0;  in mpi3mr_memset_buffers()
4657 		mrioc->op_reply_qinfo[i].num_replies = 0;  in mpi3mr_memset_buffers()
4658 		mrioc->op_reply_qinfo[i].ephase = 0;  in mpi3mr_memset_buffers()
4659 		atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0);  in mpi3mr_memset_buffers()
4660 		atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0);  in mpi3mr_memset_buffers()
4663 		mrioc->req_qinfo[i].ci = 0;  in mpi3mr_memset_buffers()
4664 		mrioc->req_qinfo[i].pi = 0;  in mpi3mr_memset_buffers()
4665 		mrioc->req_qinfo[i].num_requests = 0;  in mpi3mr_memset_buffers()
4666 		mrioc->req_qinfo[i].qid = 0;  in mpi3mr_memset_buffers()
4667 		mrioc->req_qinfo[i].reply_qid = 0;  in mpi3mr_memset_buffers()
4668 		spin_lock_init(&mrioc->req_qinfo[i].q_lock);  in mpi3mr_memset_buffers()
4672 	atomic_set(&mrioc->pend_large_data_sz, 0);  in mpi3mr_memset_buffers()
4673 	if (mrioc->throttle_groups) {  in mpi3mr_memset_buffers()
4674 		tg = mrioc->throttle_groups;  in mpi3mr_memset_buffers()
4675 		for (i = 0; i < mrioc->num_io_throttle_group; i++, tg++) {  in mpi3mr_memset_buffers()
4676 			tg->id = 0;  in mpi3mr_memset_buffers()
4677 			tg->fw_qd = 0;  in mpi3mr_memset_buffers()
4678 			tg->modified_qd = 0;  in mpi3mr_memset_buffers()
4679 			tg->io_divert = 0;  in mpi3mr_memset_buffers()
4680 			tg->need_qd_reduction = 0;  in mpi3mr_memset_buffers()
4681 			tg->high = 0;  in mpi3mr_memset_buffers()
4682 			tg->low = 0;  in mpi3mr_memset_buffers()
4683 			tg->qd_reduction = 0;  in mpi3mr_memset_buffers()
4684 			atomic_set(&tg->pend_large_data_sz, 0);  in mpi3mr_memset_buffers()
4690  * mpi3mr_free_mem - Free memory allocated for a controller
4706 	if (mrioc->sense_buf_pool) {  in mpi3mr_free_mem()
4707 		if (mrioc->sense_buf)  in mpi3mr_free_mem()
4708 			dma_pool_free(mrioc->sense_buf_pool, mrioc->sense_buf,  in mpi3mr_free_mem()
4709 			    mrioc->sense_buf_dma);  in mpi3mr_free_mem()
4710 		dma_pool_destroy(mrioc->sense_buf_pool);  in mpi3mr_free_mem()
4711 		mrioc->sense_buf = NULL;  in mpi3mr_free_mem()
4712 		mrioc->sense_buf_pool = NULL;  in mpi3mr_free_mem()
4714 	if (mrioc->sense_buf_q_pool) {  in mpi3mr_free_mem()
4715 		if (mrioc->sense_buf_q)  in mpi3mr_free_mem()
4716 			dma_pool_free(mrioc->sense_buf_q_pool,  in mpi3mr_free_mem()
4717 			    mrioc->sense_buf_q, mrioc->sense_buf_q_dma);  in mpi3mr_free_mem()
4718 		dma_pool_destroy(mrioc->sense_buf_q_pool);  in mpi3mr_free_mem()
4719 		mrioc->sense_buf_q = NULL;  in mpi3mr_free_mem()
4720 		mrioc->sense_buf_q_pool = NULL;  in mpi3mr_free_mem()
4723 	if (mrioc->reply_buf_pool) {  in mpi3mr_free_mem()
4724 		if (mrioc->reply_buf)  in mpi3mr_free_mem()
4725 			dma_pool_free(mrioc->reply_buf_pool, mrioc->reply_buf,  in mpi3mr_free_mem()
4726 			    mrioc->reply_buf_dma);  in mpi3mr_free_mem()
4727 		dma_pool_destroy(mrioc->reply_buf_pool);  in mpi3mr_free_mem()
4728 		mrioc->reply_buf = NULL;  in mpi3mr_free_mem()
4729 		mrioc->reply_buf_pool = NULL;  in mpi3mr_free_mem()
4731 	if (mrioc->reply_free_q_pool) {  in mpi3mr_free_mem()
4732 		if (mrioc->reply_free_q)  in mpi3mr_free_mem()
4733 			dma_pool_free(mrioc->reply_free_q_pool,  in mpi3mr_free_mem()
4734 			    mrioc->reply_free_q, mrioc->reply_free_q_dma);  in mpi3mr_free_mem()
4735 		dma_pool_destroy(mrioc->reply_free_q_pool);  in mpi3mr_free_mem()
4736 		mrioc->reply_free_q = NULL;  in mpi3mr_free_mem()
4737 		mrioc->reply_free_q_pool = NULL;  in mpi3mr_free_mem()
4740 	for (i = 0; i < mrioc->num_op_req_q; i++)  in mpi3mr_free_mem()
4743 	for (i = 0; i < mrioc->num_op_reply_q; i++)  in mpi3mr_free_mem()
4746 	for (i = 0; i < mrioc->intr_info_count; i++) {  in mpi3mr_free_mem()
4747 		intr_info = mrioc->intr_info + i;  in mpi3mr_free_mem()
4748 		intr_info->op_reply_q = NULL;  in mpi3mr_free_mem()
4751 	kfree(mrioc->req_qinfo);  in mpi3mr_free_mem()
4752 	mrioc->req_qinfo = NULL;  in mpi3mr_free_mem()
4753 	mrioc->num_op_req_q = 0;  in mpi3mr_free_mem()
4755 	kfree(mrioc->op_reply_qinfo);  in mpi3mr_free_mem()
4756 	mrioc->op_reply_qinfo = NULL;  in mpi3mr_free_mem()
4757 	mrioc->num_op_reply_q = 0;  in mpi3mr_free_mem()
4759 	kfree(mrioc->init_cmds.reply);  in mpi3mr_free_mem()
4760 	mrioc->init_cmds.reply = NULL;  in mpi3mr_free_mem()
4762 	kfree(mrioc->bsg_cmds.reply);  in mpi3mr_free_mem()
4763 	mrioc->bsg_cmds.reply = NULL;  in mpi3mr_free_mem()
4765 	kfree(mrioc->host_tm_cmds.reply);  in mpi3mr_free_mem()
4766 	mrioc->host_tm_cmds.reply = NULL;  in mpi3mr_free_mem()
4768 	kfree(mrioc->pel_cmds.reply);  in mpi3mr_free_mem()
4769 	mrioc->pel_cmds.reply = NULL;  in mpi3mr_free_mem()
4771 	kfree(mrioc->pel_abort_cmd.reply);  in mpi3mr_free_mem()
4772 	mrioc->pel_abort_cmd.reply = NULL;  in mpi3mr_free_mem()
4775 		kfree(mrioc->evtack_cmds[i].reply);  in mpi3mr_free_mem()
4776 		mrioc->evtack_cmds[i].reply = NULL;  in mpi3mr_free_mem()
4779 	bitmap_free(mrioc->removepend_bitmap);  in mpi3mr_free_mem()
4780 	mrioc->removepend_bitmap = NULL;  in mpi3mr_free_mem()
4782 	bitmap_free(mrioc->devrem_bitmap);  in mpi3mr_free_mem()
4783 	mrioc->devrem_bitmap = NULL;  in mpi3mr_free_mem()
4785 	bitmap_free(mrioc->evtack_cmds_bitmap);  in mpi3mr_free_mem()
4786 	mrioc->evtack_cmds_bitmap = NULL;  in mpi3mr_free_mem()
4788 	bitmap_free(mrioc->chain_bitmap);  in mpi3mr_free_mem()
4789 	mrioc->chain_bitmap = NULL;  in mpi3mr_free_mem()
4791 	kfree(mrioc->transport_cmds.reply);  in mpi3mr_free_mem()
4792 	mrioc->transport_cmds.reply = NULL;  in mpi3mr_free_mem()
4795 		kfree(mrioc->dev_rmhs_cmds[i].reply);  in mpi3mr_free_mem()
4796 		mrioc->dev_rmhs_cmds[i].reply = NULL;  in mpi3mr_free_mem()
4799 	if (mrioc->chain_buf_pool) {  in mpi3mr_free_mem()
4800 		for (i = 0; i < mrioc->chain_buf_count; i++) {  in mpi3mr_free_mem()
4801 			if (mrioc->chain_sgl_list[i].addr) {  in mpi3mr_free_mem()
4802 				dma_pool_free(mrioc->chain_buf_pool,  in mpi3mr_free_mem()
4803 				    mrioc->chain_sgl_list[i].addr,  in mpi3mr_free_mem()
4804 				    mrioc->chain_sgl_list[i].dma_addr);  in mpi3mr_free_mem()
4805 				mrioc->chain_sgl_list[i].addr = NULL;  in mpi3mr_free_mem()
4808 		dma_pool_destroy(mrioc->chain_buf_pool);  in mpi3mr_free_mem()
4809 		mrioc->chain_buf_pool = NULL;  in mpi3mr_free_mem()
4812 	kfree(mrioc->chain_sgl_list);  in mpi3mr_free_mem()
4813 	mrioc->chain_sgl_list = NULL;  in mpi3mr_free_mem()
4815 	if (mrioc->admin_reply_base) {  in mpi3mr_free_mem()
4816 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,  in mpi3mr_free_mem()
4817 		    mrioc->admin_reply_base, mrioc->admin_reply_dma);  in mpi3mr_free_mem()
4818 		mrioc->admin_reply_base = NULL;  in mpi3mr_free_mem()
4820 	if (mrioc->admin_req_base) {  in mpi3mr_free_mem()
4821 		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,  in mpi3mr_free_mem()
4822 		    mrioc->admin_req_base, mrioc->admin_req_dma);  in mpi3mr_free_mem()
4823 		mrioc->admin_req_base = NULL;  in mpi3mr_free_mem()
4826 	if (mrioc->pel_seqnum_virt) {  in mpi3mr_free_mem()
4827 		dma_free_coherent(&mrioc->pdev->dev, mrioc->pel_seqnum_sz,  in mpi3mr_free_mem()
4828 		    mrioc->pel_seqnum_virt, mrioc->pel_seqnum_dma);  in mpi3mr_free_mem()
4829 		mrioc->pel_seqnum_virt = NULL;  in mpi3mr_free_mem()
4833 		diag_buffer = &mrioc->diag_buffers[i];  in mpi3mr_free_mem()
4834 		if (diag_buffer->addr) {  in mpi3mr_free_mem()
4835 			dma_free_coherent(&mrioc->pdev->dev,  in mpi3mr_free_mem()
4836 			    diag_buffer->size, diag_buffer->addr,  in mpi3mr_free_mem()
4837 			    diag_buffer->dma_addr);  in mpi3mr_free_mem()
4838 			diag_buffer->addr = NULL;  in mpi3mr_free_mem()
4839 			diag_buffer->size = 0;  in mpi3mr_free_mem()
4840 			diag_buffer->type = 0;  in mpi3mr_free_mem()
4841 			diag_buffer->status = 0;  in mpi3mr_free_mem()
4845 	kfree(mrioc->throttle_groups);  in mpi3mr_free_mem()
4846 	mrioc->throttle_groups = NULL;  in mpi3mr_free_mem()
4848 	kfree(mrioc->logdata_buf);  in mpi3mr_free_mem()
4849 	mrioc->logdata_buf = NULL;  in mpi3mr_free_mem()
4854  * mpi3mr_issue_ioc_shutdown - shutdown controller
4869 	if (mrioc->unrecoverable) {  in mpi3mr_issue_ioc_shutdown()
4874 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_ioc_shutdown()
4881 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_ioc_shutdown()
4885 	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_ioc_shutdown()
4887 	if (mrioc->facts.shutdown_timeout)  in mpi3mr_issue_ioc_shutdown()
4888 		timeout = mrioc->facts.shutdown_timeout * 10;  in mpi3mr_issue_ioc_shutdown()
4891 		ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_ioc_shutdown()
4898 	} while (--timeout);  in mpi3mr_issue_ioc_shutdown()
4900 	ioc_status = readl(&mrioc->sysif_regs->ioc_status);  in mpi3mr_issue_ioc_shutdown()
4901 	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);  in mpi3mr_issue_ioc_shutdown()
4917  * mpi3mr_cleanup_ioc - Cleanup controller
4934 	if (!mrioc->unrecoverable && !mrioc->reset_in_progress &&  in mpi3mr_cleanup_ioc()
4935 	    !mrioc->pci_err_recovery &&  in mpi3mr_cleanup_ioc()
4948  * mpi3mr_drv_cmd_comp_reset - Flush a internal driver command
4960 	if (cmdptr->state & MPI3MR_CMD_PENDING) {  in mpi3mr_drv_cmd_comp_reset()
4961 		cmdptr->state |= MPI3MR_CMD_RESET;  in mpi3mr_drv_cmd_comp_reset()
4962 		cmdptr->state &= ~MPI3MR_CMD_PENDING;  in mpi3mr_drv_cmd_comp_reset()
4963 		if (cmdptr->is_waiting) {  in mpi3mr_drv_cmd_comp_reset()
4964 			complete(&cmdptr->done);  in mpi3mr_drv_cmd_comp_reset()
4965 			cmdptr->is_waiting = 0;  in mpi3mr_drv_cmd_comp_reset()
4966 		} else if (cmdptr->callback)  in mpi3mr_drv_cmd_comp_reset()
4967 			cmdptr->callback(mrioc, cmdptr);  in mpi3mr_drv_cmd_comp_reset()
4972  * mpi3mr_flush_drv_cmds - Flush internaldriver commands
4984 	cmdptr = &mrioc->init_cmds;  in mpi3mr_flush_drv_cmds()
4987 	cmdptr = &mrioc->cfg_cmds;  in mpi3mr_flush_drv_cmds()
4990 	cmdptr = &mrioc->bsg_cmds;  in mpi3mr_flush_drv_cmds()
4992 	cmdptr = &mrioc->host_tm_cmds;  in mpi3mr_flush_drv_cmds()
4996 		cmdptr = &mrioc->dev_rmhs_cmds[i];  in mpi3mr_flush_drv_cmds()
5001 		cmdptr = &mrioc->evtack_cmds[i];  in mpi3mr_flush_drv_cmds()
5005 	cmdptr = &mrioc->pel_cmds;  in mpi3mr_flush_drv_cmds()
5008 	cmdptr = &mrioc->pel_abort_cmd;  in mpi3mr_flush_drv_cmds()
5011 	cmdptr = &mrioc->transport_cmds;  in mpi3mr_flush_drv_cmds()
5016  * mpi3mr_pel_wait_post - Issue PEL Wait
5029 	mrioc->pel_abort_requested = false;  in mpi3mr_pel_wait_post()
5032 	drv_cmd->state = MPI3MR_CMD_PENDING;  in mpi3mr_pel_wait_post()
5033 	drv_cmd->is_waiting = 0;  in mpi3mr_pel_wait_post()
5034 	drv_cmd->callback = mpi3mr_pel_wait_complete;  in mpi3mr_pel_wait_post()
5035 	drv_cmd->ioc_status = 0;  in mpi3mr_pel_wait_post()
5036 	drv_cmd->ioc_loginfo = 0;  in mpi3mr_pel_wait_post()
5040 	pel_wait.starting_sequence_number = cpu_to_le32(mrioc->pel_newest_seqnum);  in mpi3mr_pel_wait_post()
5041 	pel_wait.locale = cpu_to_le16(mrioc->pel_locale);  in mpi3mr_pel_wait_post()
5042 	pel_wait.class = cpu_to_le16(mrioc->pel_class);  in mpi3mr_pel_wait_post()
5045 	    mrioc->pel_newest_seqnum, mrioc->pel_class, mrioc->pel_locale);  in mpi3mr_pel_wait_post()
5050 		drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_wait_post()
5051 		drv_cmd->callback = NULL;  in mpi3mr_pel_wait_post()
5052 		drv_cmd->retry_count = 0;  in mpi3mr_pel_wait_post()
5053 		mrioc->pel_enabled = false;  in mpi3mr_pel_wait_post()
5058  * mpi3mr_pel_get_seqnum_post - Issue PEL Get Sequence number
5065  * Return: 0 on success, non-zero on failure.
5075 	mrioc->pel_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_pel_get_seqnum_post()
5076 	mrioc->pel_cmds.is_waiting = 0;  in mpi3mr_pel_get_seqnum_post()
5077 	mrioc->pel_cmds.ioc_status = 0;  in mpi3mr_pel_get_seqnum_post()
5078 	mrioc->pel_cmds.ioc_loginfo = 0;  in mpi3mr_pel_get_seqnum_post()
5079 	mrioc->pel_cmds.callback = mpi3mr_pel_get_seqnum_complete;  in mpi3mr_pel_get_seqnum_post()
5084 	    mrioc->pel_seqnum_sz, mrioc->pel_seqnum_dma);  in mpi3mr_pel_get_seqnum_post()
5090 			drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_get_seqnum_post()
5091 			drv_cmd->callback = NULL;  in mpi3mr_pel_get_seqnum_post()
5092 			drv_cmd->retry_count = 0;  in mpi3mr_pel_get_seqnum_post()
5094 		mrioc->pel_enabled = false;  in mpi3mr_pel_get_seqnum_post()
5101  * mpi3mr_pel_wait_complete - PELWait Completion callback
5120 	if (drv_cmd->state & MPI3MR_CMD_RESET)  in mpi3mr_pel_wait_complete()
5123 	ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;  in mpi3mr_pel_wait_complete()
5126 			__func__, ioc_status, drv_cmd->ioc_loginfo);  in mpi3mr_pel_wait_complete()
5129 		    ioc_status, drv_cmd->ioc_loginfo);  in mpi3mr_pel_wait_complete()
5133 	if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)  in mpi3mr_pel_wait_complete()
5134 		pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;  in mpi3mr_pel_wait_complete()
5142 	pe_log_status = le16_to_cpu(pel_reply->pe_log_status);  in mpi3mr_pel_wait_complete()
5154 		if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {  in mpi3mr_pel_wait_complete()
5155 			drv_cmd->retry_count++;  in mpi3mr_pel_wait_complete()
5157 			    drv_cmd->retry_count);  in mpi3mr_pel_wait_complete()
5163 		    drv_cmd->retry_count);  in mpi3mr_pel_wait_complete()
5167 	if (!mrioc->pel_abort_requested) {  in mpi3mr_pel_wait_complete()
5168 		mrioc->pel_cmds.retry_count = 0;  in mpi3mr_pel_wait_complete()
5169 		mpi3mr_pel_get_seqnum_post(mrioc, &mrioc->pel_cmds);  in mpi3mr_pel_wait_complete()
5174 	mrioc->pel_enabled = false;  in mpi3mr_pel_wait_complete()
5176 	drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_wait_complete()
5177 	drv_cmd->callback = NULL;  in mpi3mr_pel_wait_complete()
5178 	drv_cmd->retry_count = 0;  in mpi3mr_pel_wait_complete()
5182  * mpi3mr_pel_get_seqnum_complete - PELGetSeqNum Completion callback
5200 	pel_seqnum_virt = (struct mpi3_pel_seq *)mrioc->pel_seqnum_virt;  in mpi3mr_pel_get_seqnum_complete()
5202 	if (drv_cmd->state & MPI3MR_CMD_RESET)  in mpi3mr_pel_get_seqnum_complete()
5205 	ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;  in mpi3mr_pel_get_seqnum_complete()
5209 		    ioc_status, drv_cmd->ioc_loginfo);  in mpi3mr_pel_get_seqnum_complete()
5213 	if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)  in mpi3mr_pel_get_seqnum_complete()
5214 		pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;  in mpi3mr_pel_get_seqnum_complete()
5221 	if (le16_to_cpu(pel_reply->pe_log_status) != MPI3_PEL_STATUS_SUCCESS) {  in mpi3mr_pel_get_seqnum_complete()
5224 		    le16_to_cpu(pel_reply->pe_log_status));  in mpi3mr_pel_get_seqnum_complete()
5229 		if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {  in mpi3mr_pel_get_seqnum_complete()
5230 			drv_cmd->retry_count++;  in mpi3mr_pel_get_seqnum_complete()
5233 			    drv_cmd->retry_count);  in mpi3mr_pel_get_seqnum_complete()
5240 		    drv_cmd->retry_count);  in mpi3mr_pel_get_seqnum_complete()
5243 	mrioc->pel_newest_seqnum = le32_to_cpu(pel_seqnum_virt->newest) + 1;  in mpi3mr_pel_get_seqnum_complete()
5244 	drv_cmd->retry_count = 0;  in mpi3mr_pel_get_seqnum_complete()
5249 	mrioc->pel_enabled = false;  in mpi3mr_pel_get_seqnum_complete()
5251 	drv_cmd->state = MPI3MR_CMD_NOTUSED;  in mpi3mr_pel_get_seqnum_complete()
5252 	drv_cmd->callback = NULL;  in mpi3mr_pel_get_seqnum_complete()
5253 	drv_cmd->retry_count = 0;  in mpi3mr_pel_get_seqnum_complete()
5257  * mpi3mr_check_op_admin_proc -
5265  * Return: 0 on success, non-zero on failure.
5278 		if (atomic_read(&mrioc->admin_reply_q_in_use) == 1)  in mpi3mr_check_op_admin_proc()
5284 			for (i = 0; i < mrioc->num_queues; i++) {  in mpi3mr_check_op_admin_proc()
5285 				if (atomic_read(&mrioc->op_reply_qinfo[i].in_use) == 1) {  in mpi3mr_check_op_admin_proc()
5306  * mpi3mr_soft_reset_handler - Reset the controller
5324  * Return: 0 on success, non-zero on failure.
5337 	    mrioc->diagsave_timeout);  in mpi3mr_soft_reset_handler()
5338 	while (mrioc->diagsave_timeout)  in mpi3mr_soft_reset_handler()
5345 	if (!mutex_trylock(&mrioc->reset_mutex)) {  in mpi3mr_soft_reset_handler()
5351 		} while (mrioc->reset_in_progress == 1);  in mpi3mr_soft_reset_handler()
5354 		    mrioc->prev_reset_result,  in mpi3mr_soft_reset_handler()
5356 		return mrioc->prev_reset_result;  in mpi3mr_soft_reset_handler()
5361 	mrioc->device_refresh_on = 0;  in mpi3mr_soft_reset_handler()
5362 	mrioc->reset_in_progress = 1;  in mpi3mr_soft_reset_handler()
5363 	mrioc->stop_bsgs = 1;  in mpi3mr_soft_reset_handler()
5364 	mrioc->prev_reset_result = -1;  in mpi3mr_soft_reset_handler()
5376 			mrioc->event_masks[i] = -1;  in mpi3mr_soft_reset_handler()
5385 	mrioc->io_admin_reset_sync = 1;  in mpi3mr_soft_reset_handler()
5392 			trigger_data.fault = (readl(&mrioc->sysif_regs->fault) &  in mpi3mr_soft_reset_handler()
5396 				    readl(&mrioc->sysif_regs->host_diagnostic);  in mpi3mr_soft_reset_handler()
5401 			} while (--timeout);  in mpi3mr_soft_reset_handler()
5423 	if (mrioc->num_io_throttle_group !=  in mpi3mr_soft_reset_handler()
5424 	    mrioc->facts.max_io_throttle_group) {  in mpi3mr_soft_reset_handler()
5427 		    mrioc->num_io_throttle_group,  in mpi3mr_soft_reset_handler()
5428 		    mrioc->facts.max_io_throttle_group);  in mpi3mr_soft_reset_handler()
5429 		retval = -EPERM;  in mpi3mr_soft_reset_handler()
5435 	bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);  in mpi3mr_soft_reset_handler()
5436 	bitmap_clear(mrioc->removepend_bitmap, 0,  in mpi3mr_soft_reset_handler()
5437 		     mrioc->dev_handle_bitmap_bits);  in mpi3mr_soft_reset_handler()
5438 	bitmap_clear(mrioc->evtack_cmds_bitmap, 0, MPI3MR_NUM_EVTACKCMD);  in mpi3mr_soft_reset_handler()
5444 	if (mrioc->prepare_for_reset) {  in mpi3mr_soft_reset_handler()
5445 		mrioc->prepare_for_reset = 0;  in mpi3mr_soft_reset_handler()
5446 		mrioc->prepare_for_reset_timeout_counter = 0;  in mpi3mr_soft_reset_handler()
5450 	mrioc->fw_release_trigger_active = false;  in mpi3mr_soft_reset_handler()
5451 	mrioc->trace_release_trigger_active = false;  in mpi3mr_soft_reset_handler()
5452 	mrioc->snapdump_trigger_active = false;  in mpi3mr_soft_reset_handler()
5461 		    mrioc->name, reset_reason);  in mpi3mr_soft_reset_handler()
5468 		mrioc->diagsave_timeout = 0;  in mpi3mr_soft_reset_handler()
5469 		mrioc->reset_in_progress = 0;  in mpi3mr_soft_reset_handler()
5470 		mrioc->pel_abort_requested = 0;  in mpi3mr_soft_reset_handler()
5471 		if (mrioc->pel_enabled) {  in mpi3mr_soft_reset_handler()
5472 			mrioc->pel_cmds.retry_count = 0;  in mpi3mr_soft_reset_handler()
5473 			mpi3mr_pel_wait_post(mrioc, &mrioc->pel_cmds);  in mpi3mr_soft_reset_handler()
5476 		mrioc->device_refresh_on = 0;  in mpi3mr_soft_reset_handler()
5478 		mrioc->ts_update_counter = 0;  in mpi3mr_soft_reset_handler()
5479 		spin_lock_irqsave(&mrioc->watchdog_lock, flags);  in mpi3mr_soft_reset_handler()
5480 		if (mrioc->watchdog_work_q)  in mpi3mr_soft_reset_handler()
5481 			queue_delayed_work(mrioc->watchdog_work_q,  in mpi3mr_soft_reset_handler()
5482 			    &mrioc->watchdog_work,  in mpi3mr_soft_reset_handler()
5484 		spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);  in mpi3mr_soft_reset_handler()
5485 		mrioc->stop_bsgs = 0;  in mpi3mr_soft_reset_handler()
5486 		if (mrioc->pel_enabled)  in mpi3mr_soft_reset_handler()
5491 		mrioc->device_refresh_on = 0;  in mpi3mr_soft_reset_handler()
5492 		mrioc->unrecoverable = 1;  in mpi3mr_soft_reset_handler()
5493 		mrioc->reset_in_progress = 0;  in mpi3mr_soft_reset_handler()
5494 		mrioc->stop_bsgs = 0;  in mpi3mr_soft_reset_handler()
5495 		retval = -1;  in mpi3mr_soft_reset_handler()
5498 	mrioc->prev_reset_result = retval;  in mpi3mr_soft_reset_handler()
5499 	mutex_unlock(&mrioc->reset_mutex);  in mpi3mr_soft_reset_handler()
5506  * mpi3mr_post_cfg_req - Issue config requests and wait
5520  * Return: 0 on success, non-zero on failure.
5527 	mutex_lock(&mrioc->cfg_cmds.mutex);  in mpi3mr_post_cfg_req()
5528 	if (mrioc->cfg_cmds.state & MPI3MR_CMD_PENDING) {  in mpi3mr_post_cfg_req()
5529 		retval = -1;  in mpi3mr_post_cfg_req()
5531 		mutex_unlock(&mrioc->cfg_cmds.mutex);  in mpi3mr_post_cfg_req()
5534 	mrioc->cfg_cmds.state = MPI3MR_CMD_PENDING;  in mpi3mr_post_cfg_req()
5535 	mrioc->cfg_cmds.is_waiting = 1;  in mpi3mr_post_cfg_req()
5536 	mrioc->cfg_cmds.callback = NULL;  in mpi3mr_post_cfg_req()
5537 	mrioc->cfg_cmds.ioc_status = 0;  in mpi3mr_post_cfg_req()
5538 	mrioc->cfg_cmds.ioc_loginfo = 0;  in mpi3mr_post_cfg_req()
5540 	cfg_req->host_tag = cpu_to_le16(MPI3MR_HOSTTAG_CFG_CMDS);  in mpi3mr_post_cfg_req()
5541 	cfg_req->function = MPI3_FUNCTION_CONFIG;  in mpi3mr_post_cfg_req()
5543 	init_completion(&mrioc->cfg_cmds.done);  in mpi3mr_post_cfg_req()
5545 	if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)  in mpi3mr_post_cfg_req()
5553 	wait_for_completion_timeout(&mrioc->cfg_cmds.done, (timeout * HZ));  in mpi3mr_post_cfg_req()
5554 	if (!(mrioc->cfg_cmds.state & MPI3MR_CMD_COMPLETE)) {  in mpi3mr_post_cfg_req()
5558 		retval = -1;  in mpi3mr_post_cfg_req()
5561 	*ioc_status = mrioc->cfg_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK;  in mpi3mr_post_cfg_req()
5565 		    *ioc_status, mrioc->cfg_cmds.ioc_loginfo);  in mpi3mr_post_cfg_req()
5568 	mrioc->cfg_cmds.state = MPI3MR_CMD_NOTUSED;  in mpi3mr_post_cfg_req()
5569 	mutex_unlock(&mrioc->cfg_cmds.mutex);  in mpi3mr_post_cfg_req()
5576  * mpi3mr_process_cfg_req - config page request processor
5609  * Return: 0 on success, non-zero on failure.
5617 	int retval = -1;  in mpi3mr_process_cfg_req()
5623 	if (cfg_req->action == MPI3_CONFIG_ACTION_PAGE_HEADER)  in mpi3mr_process_cfg_req()
5628 			    cfg_req->action, cfg_req->page_type,  in mpi3mr_process_cfg_req()
5629 			    cfg_req->page_number);  in mpi3mr_process_cfg_req()
5632 		switch (cfg_hdr->page_attribute & MPI3_CONFIG_PAGEATTR_MASK) {  in mpi3mr_process_cfg_req()
5634 			if (cfg_req->action  in mpi3mr_process_cfg_req()
5639 			if ((cfg_req->action ==  in mpi3mr_process_cfg_req()
5641 			    (cfg_req->action ==  in mpi3mr_process_cfg_req()
5652 			    cfg_req->action, cfg_req->page_type,  in mpi3mr_process_cfg_req()
5653 			    cfg_req->page_number, cfg_hdr->page_attribute);  in mpi3mr_process_cfg_req()
5656 		mem_desc.size = le16_to_cpu(cfg_hdr->page_length) * 4;  in mpi3mr_process_cfg_req()
5657 		cfg_req->page_length = cfg_hdr->page_length;  in mpi3mr_process_cfg_req()
5658 		cfg_req->page_version = cfg_hdr->page_version;  in mpi3mr_process_cfg_req()
5661 	mem_desc.addr = dma_alloc_coherent(&mrioc->pdev->dev,  in mpi3mr_process_cfg_req()
5667 	mpi3mr_add_sg_single(&cfg_req->sgl, sgl_flags, mem_desc.size,  in mpi3mr_process_cfg_req()
5670 	if ((cfg_req->action == MPI3_CONFIG_ACTION_WRITE_PERSISTENT) ||  in mpi3mr_process_cfg_req()
5671 	    (cfg_req->action == MPI3_CONFIG_ACTION_WRITE_CURRENT)) {  in mpi3mr_process_cfg_req()
5675 		if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)  in mpi3mr_process_cfg_req()
5684 	    (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_PERSISTENT) &&  in mpi3mr_process_cfg_req()
5685 	    (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_CURRENT)) {  in mpi3mr_process_cfg_req()
5689 		if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)  in mpi3mr_process_cfg_req()
5695 		dma_free_coherent(&mrioc->pdev->dev, mem_desc.size,  in mpi3mr_process_cfg_req()
5704  * mpi3mr_cfg_get_dev_pg0 - Read current device page0
5718  * Return: 0 on success, non-zero on failure.
5758 	return -1;  in mpi3mr_cfg_get_dev_pg0()
5763  * mpi3mr_cfg_get_sas_phy_pg0 - Read current SAS Phy page0
5777  * Return: 0 on success, non-zero on failure.
5818 	return -1;  in mpi3mr_cfg_get_sas_phy_pg0()
5822  * mpi3mr_cfg_get_sas_phy_pg1 - Read current SAS Phy page1
5836  * Return: 0 on success, non-zero on failure.
5877 	return -1;  in mpi3mr_cfg_get_sas_phy_pg1()
5882  * mpi3mr_cfg_get_sas_exp_pg0 - Read current SAS Expander page0
5896  * Return: 0 on success, non-zero on failure.
5938 	return -1;  in mpi3mr_cfg_get_sas_exp_pg0()
5942  * mpi3mr_cfg_get_sas_exp_pg1 - Read current SAS Expander page1
5956  * Return: 0 on success, non-zero on failure.
5998 	return -1;  in mpi3mr_cfg_get_sas_exp_pg1()
6002  * mpi3mr_cfg_get_enclosure_pg0 - Read current Enclosure page0
6016  * Return: 0 on success, non-zero on failure.
6057 	return -1;  in mpi3mr_cfg_get_enclosure_pg0()
6062  * mpi3mr_cfg_get_sas_io_unit_pg0 - Read current SASIOUnit page0
6071  * Return: 0 on success, non-zero on failure.
6114 	return -1;  in mpi3mr_cfg_get_sas_io_unit_pg0()
6118  * mpi3mr_cfg_get_sas_io_unit_pg1 - Read current SASIOUnit page1
6127  * Return: 0 on success, non-zero on failure.
6170 	return -1;  in mpi3mr_cfg_get_sas_io_unit_pg1()
6174  * mpi3mr_cfg_set_sas_io_unit_pg1 - Write SASIOUnit page1
6184  * Return: 0 on success, non-zero on failure.
6239 	return -1;  in mpi3mr_cfg_set_sas_io_unit_pg1()
6243  * mpi3mr_cfg_get_driver_pg1 - Read current Driver page1
6252  * Return: 0 on success, non-zero on failure.
6295 	return -1;  in mpi3mr_cfg_get_driver_pg1()
6299  * mpi3mr_cfg_get_driver_pg2 - Read current driver page2
6309  * Return: 0 on success, non-zero on failure.
6355 	return -1;  in mpi3mr_cfg_get_driver_pg2()