87 static int ena_com_admin_init_sq(struct ena_com_admin_queue *admin_queue)  in ena_com_admin_init_sq()  argument
89 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_admin_init_sq()
90 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_init_sq()
91 	u16 size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_init_sq()
93 	sq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size, &sq->dma_addr, GFP_KERNEL);  in ena_com_admin_init_sq()
109 static int ena_com_admin_init_cq(struct ena_com_admin_queue *admin_queue)  in ena_com_admin_init_cq()  argument
111 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_admin_init_cq()
112 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_init_cq()
113 	u16 size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_init_cq()
115 	cq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size, &cq->dma_addr, GFP_KERNEL);  in ena_com_admin_init_cq()
177 static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *admin_queue,  in get_comp_ctxt()  argument
180 	if (unlikely(command_id >= admin_queue->q_depth)) {  in get_comp_ctxt()
181 		netdev_err(admin_queue->ena_dev->net_device,  in get_comp_ctxt()
183 			   command_id, admin_queue->q_depth);  in get_comp_ctxt()
187 	if (unlikely(!admin_queue->comp_ctx)) {  in get_comp_ctxt()
188 		netdev_err(admin_queue->ena_dev->net_device, "Completion context is NULL\n");  in get_comp_ctxt()
192 	if (unlikely(admin_queue->comp_ctx[command_id].occupied && capture)) {  in get_comp_ctxt()
193 		netdev_err(admin_queue->ena_dev->net_device, "Completion context is occupied\n");  in get_comp_ctxt()
198 		atomic_inc(&admin_queue->outstanding_cmds);  in get_comp_ctxt()
199 		admin_queue->comp_ctx[command_id].occupied = true;  in get_comp_ctxt()
202 	return &admin_queue->comp_ctx[command_id];  in get_comp_ctxt()
205 static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in __ena_com_submit_admin_cmd()  argument
216 	queue_size_mask = admin_queue->q_depth - 1;  in __ena_com_submit_admin_cmd()
218 	tail_masked = admin_queue->sq.tail & queue_size_mask;  in __ena_com_submit_admin_cmd()
221 	cnt = (u16)atomic_read(&admin_queue->outstanding_cmds);  in __ena_com_submit_admin_cmd()
222 	if (cnt >= admin_queue->q_depth) {  in __ena_com_submit_admin_cmd()
223 		netdev_dbg(admin_queue->ena_dev->net_device, "Admin queue is full.\n");  in __ena_com_submit_admin_cmd()
224 		admin_queue->stats.out_of_space++;  in __ena_com_submit_admin_cmd()
228 	cmd_id = admin_queue->curr_cmd_id;  in __ena_com_submit_admin_cmd()
230 	cmd->aq_common_descriptor.flags |= admin_queue->sq.phase &  in __ena_com_submit_admin_cmd()
236 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, true);  in __ena_com_submit_admin_cmd()
247 	memcpy(&admin_queue->sq.entries[tail_masked], cmd, cmd_size_in_bytes);  in __ena_com_submit_admin_cmd()
249 	admin_queue->curr_cmd_id = (admin_queue->curr_cmd_id + 1) &  in __ena_com_submit_admin_cmd()
252 	admin_queue->sq.tail++;  in __ena_com_submit_admin_cmd()
253 	admin_queue->stats.submitted_cmd++;  in __ena_com_submit_admin_cmd()
255 	if (unlikely((admin_queue->sq.tail & queue_size_mask) == 0))  in __ena_com_submit_admin_cmd()
256 		admin_queue->sq.phase = !admin_queue->sq.phase;  in __ena_com_submit_admin_cmd()
258 	writel(admin_queue->sq.tail, admin_queue->sq.db_addr);  in __ena_com_submit_admin_cmd()
263 static int ena_com_init_comp_ctxt(struct ena_com_admin_queue *admin_queue)  in ena_com_init_comp_ctxt()  argument
265 	struct ena_com_dev *ena_dev = admin_queue->ena_dev;  in ena_com_init_comp_ctxt()
266 	size_t size = admin_queue->q_depth * sizeof(struct ena_comp_ctx);  in ena_com_init_comp_ctxt()
270 	admin_queue->comp_ctx = devm_kzalloc(admin_queue->q_dmadev, size, GFP_KERNEL);  in ena_com_init_comp_ctxt()
271 	if (unlikely(!admin_queue->comp_ctx)) {  in ena_com_init_comp_ctxt()
276 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_init_comp_ctxt()
277 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_init_comp_ctxt()
285 static struct ena_comp_ctx *ena_com_submit_admin_cmd(struct ena_com_admin_queue *admin_queue,  in ena_com_submit_admin_cmd()  argument
294 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
295 	if (unlikely(!admin_queue->running_state)) {  in ena_com_submit_admin_cmd()
296 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
299 	comp_ctx = __ena_com_submit_admin_cmd(admin_queue, cmd,  in ena_com_submit_admin_cmd()
304 		admin_queue->running_state = false;  in ena_com_submit_admin_cmd()
305 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_submit_admin_cmd()
423 static void ena_com_handle_single_admin_completion(struct ena_com_admin_queue *admin_queue,  in ena_com_handle_single_admin_completion()  argument
432 	comp_ctx = get_comp_ctxt(admin_queue, cmd_id, false);  in ena_com_handle_single_admin_completion()
434 		netdev_err(admin_queue->ena_dev->net_device,  in ena_com_handle_single_admin_completion()
436 		admin_queue->running_state = false;  in ena_com_handle_single_admin_completion()
446 	if (!admin_queue->polling)  in ena_com_handle_single_admin_completion()
450 static void ena_com_handle_admin_completion(struct ena_com_admin_queue *admin_queue)  in ena_com_handle_admin_completion()  argument
457 	head_masked = admin_queue->cq.head & (admin_queue->q_depth - 1);  in ena_com_handle_admin_completion()
458 	phase = admin_queue->cq.phase;  in ena_com_handle_admin_completion()
460 	cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
469 		ena_com_handle_single_admin_completion(admin_queue, cqe);  in ena_com_handle_admin_completion()
473 		if (unlikely(head_masked == admin_queue->q_depth)) {  in ena_com_handle_admin_completion()
478 		cqe = &admin_queue->cq.entries[head_masked];  in ena_com_handle_admin_completion()
481 	admin_queue->cq.head += comp_num;  in ena_com_handle_admin_completion()
482 	admin_queue->cq.phase = phase;  in ena_com_handle_admin_completion()
483 	admin_queue->sq.head += comp_num;  in ena_com_handle_admin_completion()
484 	admin_queue->stats.completed_cmd += comp_num;  in ena_com_handle_admin_completion()
487 static int ena_com_comp_status_to_errno(struct ena_com_admin_queue *admin_queue,  in ena_com_comp_status_to_errno()  argument
491 		netdev_err(admin_queue->ena_dev->net_device, "Admin command failed[%u]\n",  in ena_com_comp_status_to_errno()
522 						     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_polling()  argument
529 	timeout = jiffies + usecs_to_jiffies(admin_queue->completion_timeout);  in ena_com_wait_and_process_admin_cq_polling()
532 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
533 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_polling()
534 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
540 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_wait_and_process_admin_cq_polling()
543 			spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
544 			admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_polling()
545 			admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_polling()
546 			spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
553 						 admin_queue->ena_dev->ena_min_poll_delay_us);  in ena_com_wait_and_process_admin_cq_polling()
557 		netdev_err(admin_queue->ena_dev->net_device, "Command was aborted\n");  in ena_com_wait_and_process_admin_cq_polling()
558 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
559 		admin_queue->stats.aborted_cmd++;  in ena_com_wait_and_process_admin_cq_polling()
560 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_polling()
567 	ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);  in ena_com_wait_and_process_admin_cq_polling()
569 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_polling()
581 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_llq()  local
588 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_llq()
602 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_llq()
745 							struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq_interrupts()  argument
751 				    usecs_to_jiffies(admin_queue->completion_timeout));  in ena_com_wait_and_process_admin_cq_interrupts()
759 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
760 		ena_com_handle_admin_completion(admin_queue);  in ena_com_wait_and_process_admin_cq_interrupts()
761 		admin_queue->stats.no_completion++;  in ena_com_wait_and_process_admin_cq_interrupts()
762 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_and_process_admin_cq_interrupts()
765 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_wait_and_process_admin_cq_interrupts()
769 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_wait_and_process_admin_cq_interrupts()
773 		admin_queue->running_state = false;  in ena_com_wait_and_process_admin_cq_interrupts()
778 	ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status);  in ena_com_wait_and_process_admin_cq_interrupts()
780 	comp_ctxt_release(admin_queue, comp_ctx);  in ena_com_wait_and_process_admin_cq_interrupts()
852 					     struct ena_com_admin_queue *admin_queue)  in ena_com_wait_and_process_admin_cq()  argument
854 	if (admin_queue->polling)  in ena_com_wait_and_process_admin_cq()
856 								 admin_queue);  in ena_com_wait_and_process_admin_cq()
859 							    admin_queue);  in ena_com_wait_and_process_admin_cq()
865 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_sq()  local
885 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_sq()
975 	struct ena_com_admin_queue *admin_queue;  in ena_com_get_feature_ex()  local
985 	admin_queue = &ena_dev->admin_queue;  in ena_com_get_feature_ex()
1007 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_get_feature_ex()
1170 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_sq()  local
1212 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_sq()
1291 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue,  in ena_com_execute_admin_command()  argument
1300 	comp_ctx = ena_com_submit_admin_cmd(admin_queue, cmd, cmd_size,  in ena_com_execute_admin_command()
1305 			netdev_dbg(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1308 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1314 	ret = ena_com_wait_and_process_admin_cq(comp_ctx, admin_queue);  in ena_com_execute_admin_command()
1316 		if (admin_queue->running_state)  in ena_com_execute_admin_command()
1317 			netdev_err(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1320 			netdev_dbg(admin_queue->ena_dev->net_device,  in ena_com_execute_admin_command()
1329 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_create_io_cq()  local
1354 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_create_io_cq()
1397 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_abort_admin_commands()  local
1401 	if (!admin_queue->comp_ctx)  in ena_com_abort_admin_commands()
1404 	for (i = 0; i < admin_queue->q_depth; i++) {  in ena_com_abort_admin_commands()
1405 		comp_ctx = get_comp_ctxt(admin_queue, i, false);  in ena_com_abort_admin_commands()
1417 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_wait_for_abort_completion()  local
1421 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1422 	while (atomic_read(&admin_queue->outstanding_cmds) != 0) {  in ena_com_wait_for_abort_completion()
1423 		spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1425 		spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1427 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_wait_for_abort_completion()
1433 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_destroy_io_cq()  local
1443 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_destroy_io_cq()
1457 	return ena_dev->admin_queue.running_state;  in ena_com_get_admin_running_state()
1462 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_admin_running_state()  local
1465 	spin_lock_irqsave(&admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1466 	ena_dev->admin_queue.running_state = state;  in ena_com_set_admin_running_state()
1467 	spin_unlock_irqrestore(&admin_queue->q_lock, flags);  in ena_com_set_admin_running_state()
1484 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_aenq_config()  local
1504 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_aenq_config()
1511 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_aenq_config()
1596 				      struct ena_com_admin_queue *admin_queue)  in ena_com_free_ena_admin_queue_comp_ctx()  argument
1599 	if (!admin_queue->comp_ctx)  in ena_com_free_ena_admin_queue_comp_ctx()
1602 	devm_kfree(ena_dev->dmadev, admin_queue->comp_ctx);  in ena_com_free_ena_admin_queue_comp_ctx()
1604 	admin_queue->comp_ctx = NULL;  in ena_com_free_ena_admin_queue_comp_ctx()
1609 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_destroy()  local
1610 	struct ena_com_admin_cq *cq = &admin_queue->cq;  in ena_com_admin_destroy()
1611 	struct ena_com_admin_sq *sq = &admin_queue->sq;  in ena_com_admin_destroy()
1615 	ena_com_free_ena_admin_queue_comp_ctx(ena_dev, admin_queue);  in ena_com_admin_destroy()
1617 	size = ADMIN_SQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1622 	size = ADMIN_CQ_SIZE(admin_queue->q_depth);  in ena_com_admin_destroy()
1641 	ena_dev->admin_queue.polling = polling;  in ena_com_set_admin_polling_mode()
1702 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_admin_init()  local
1718 	admin_queue->q_depth = ENA_ADMIN_QUEUE_DEPTH;  in ena_com_admin_init()
1720 	admin_queue->q_dmadev = ena_dev->dmadev;  in ena_com_admin_init()
1721 	admin_queue->polling = false;  in ena_com_admin_init()
1722 	admin_queue->curr_cmd_id = 0;  in ena_com_admin_init()
1724 	atomic_set(&admin_queue->outstanding_cmds, 0);  in ena_com_admin_init()
1726 	spin_lock_init(&admin_queue->q_lock);  in ena_com_admin_init()
1728 	ret = ena_com_init_comp_ctxt(admin_queue);  in ena_com_admin_init()
1732 	ret = ena_com_admin_init_sq(admin_queue);  in ena_com_admin_init()
1736 	ret = ena_com_admin_init_cq(admin_queue);  in ena_com_admin_init()
1740 	admin_queue->sq.db_addr = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar +  in ena_com_admin_init()
1743 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1744 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->sq.dma_addr);  in ena_com_admin_init()
1749 	addr_low = ENA_DMA_ADDR_TO_UINT32_LOW(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1750 	addr_high = ENA_DMA_ADDR_TO_UINT32_HIGH(admin_queue->cq.dma_addr);  in ena_com_admin_init()
1756 	aq_caps |= admin_queue->q_depth & ENA_REGS_AQ_CAPS_AQ_DEPTH_MASK;  in ena_com_admin_init()
1762 	acq_caps |= admin_queue->q_depth & ENA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;  in ena_com_admin_init()
1773 	admin_queue->ena_dev = ena_dev;  in ena_com_admin_init()
1774 	admin_queue->running_state = true;  in ena_com_admin_init()
1875 	struct ena_com_admin_queue *admin_queue;  in ena_get_dev_stats()  local
1878 	admin_queue = &ena_dev->admin_queue;  in ena_get_dev_stats()
1884 	ret = ena_com_execute_admin_command(admin_queue,  in ena_get_dev_stats()
2005 	ena_com_handle_admin_completion(&ena_dev->admin_queue);  in ena_com_admin_q_comp_intr_handler()
2137 		ena_dev->admin_queue.completion_timeout = timeout * 100000;  in ena_com_dev_reset()
2139 		ena_dev->admin_queue.completion_timeout = ADMIN_CMD_TIMEOUT_US;  in ena_com_dev_reset()
2232 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_dev_mtu()  local
2243 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_dev_mtu()
2250 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_dev_mtu()
2264 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_function()  local
2308 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_function()
2435 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_set_hash_ctrl()  local
2467 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_hash_ctrl()
2596 	struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue;  in ena_com_indirect_table_set()  local
2635 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_indirect_table_set()
2807 	struct ena_com_admin_queue *admin_queue;  in ena_com_set_host_attributes()  local
2818 	admin_queue = &ena_dev->admin_queue;  in ena_com_set_host_attributes()
2841 	ret = ena_com_execute_admin_command(admin_queue,  in ena_com_set_host_attributes()