Lines Matching +full:combined +full:- +full:power +full:- +full:req
1 // SPDX-License-Identifier: GPL-2.0
7 * Copyright (c) 2011-2014, Intel Corporation.
9 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
14 #include <linux/blk-mq.h>
16 #include <linux/dma-mapping.h>
19 #include <linux/io-64-nonatomic-lo-hi.h>
69 * cannot be higher than 0x40 which effectively limits the combined
76 #define APPLE_NVME_AQ_MQ_TAG_DEPTH (APPLE_NVME_AQ_DEPTH - 1)
91 * An exception to this are Apple's vendor-specific commands (opcode 0xD8 on the
159 struct nvme_request req; member
210 if (q->is_adminq) in queue_to_apple_nvme()
218 if (q->is_adminq) in apple_nvme_queue_depth()
228 dev_warn(anv->dev, "RTKit crashed; unable to recover without a reboot"); in apple_nvme_rtkit_crashed()
229 nvme_reset_ctrl(&anv->ctrl); in apple_nvme_rtkit_crashed()
238 if (bfr->iova) in apple_nvme_sart_dma_setup()
239 return -EINVAL; in apple_nvme_sart_dma_setup()
240 if (!bfr->size) in apple_nvme_sart_dma_setup()
241 return -EINVAL; in apple_nvme_sart_dma_setup()
243 bfr->buffer = in apple_nvme_sart_dma_setup()
244 dma_alloc_coherent(anv->dev, bfr->size, &bfr->iova, GFP_KERNEL); in apple_nvme_sart_dma_setup()
245 if (!bfr->buffer) in apple_nvme_sart_dma_setup()
246 return -ENOMEM; in apple_nvme_sart_dma_setup()
248 ret = apple_sart_add_allowed_region(anv->sart, bfr->iova, bfr->size); in apple_nvme_sart_dma_setup()
250 dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova); in apple_nvme_sart_dma_setup()
251 bfr->buffer = NULL; in apple_nvme_sart_dma_setup()
252 return -ENOMEM; in apple_nvme_sart_dma_setup()
263 apple_sart_remove_allowed_region(anv->sart, bfr->iova, bfr->size); in apple_nvme_sart_dma_destroy()
264 dma_free_coherent(anv->dev, bfr->size, bfr->buffer, bfr->iova); in apple_nvme_sart_dma_destroy()
277 writel(tag, anv->mmio_nvme + APPLE_NVMMU_TCB_INVAL); in apple_nvmmu_inval()
278 if (readl(anv->mmio_nvme + APPLE_NVMMU_TCB_STAT)) in apple_nvmmu_inval()
279 dev_warn_ratelimited(anv->dev, in apple_nvmmu_inval()
287 u32 tag = nvme_tag_from_cid(cmd->common.command_id); in apple_nvme_submit_cmd()
288 struct apple_nvmmu_tcb *tcb = &q->tcbs[tag]; in apple_nvme_submit_cmd()
290 tcb->opcode = cmd->common.opcode; in apple_nvme_submit_cmd()
291 tcb->prp1 = cmd->common.dptr.prp1; in apple_nvme_submit_cmd()
292 tcb->prp2 = cmd->common.dptr.prp2; in apple_nvme_submit_cmd()
293 tcb->length = cmd->rw.length; in apple_nvme_submit_cmd()
294 tcb->command_id = tag; in apple_nvme_submit_cmd()
297 tcb->dma_flags = APPLE_ANS_TCB_DMA_TO_DEVICE; in apple_nvme_submit_cmd()
299 tcb->dma_flags = APPLE_ANS_TCB_DMA_FROM_DEVICE; in apple_nvme_submit_cmd()
301 memcpy(&q->sqes[tag], cmd, sizeof(*cmd)); in apple_nvme_submit_cmd()
313 spin_lock_irq(&anv->lock); in apple_nvme_submit_cmd()
314 writel(tag, q->sq_db); in apple_nvme_submit_cmd()
315 spin_unlock_irq(&anv->lock); in apple_nvme_submit_cmd()
328 const int npages = DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8); in apple_nvme_iod_alloc_size()
335 static void **apple_nvme_iod_list(struct request *req) in apple_nvme_iod_list() argument
337 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_iod_list()
339 return (void **)(iod->sg + blk_rq_nr_phys_segments(req)); in apple_nvme_iod_list()
342 static void apple_nvme_free_prps(struct apple_nvme *anv, struct request *req) in apple_nvme_free_prps() argument
344 const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1; in apple_nvme_free_prps()
345 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_free_prps()
346 dma_addr_t dma_addr = iod->first_dma; in apple_nvme_free_prps()
349 for (i = 0; i < iod->npages; i++) { in apple_nvme_free_prps()
350 __le64 *prp_list = apple_nvme_iod_list(req)[i]; in apple_nvme_free_prps()
353 dma_pool_free(anv->prp_page_pool, prp_list, dma_addr); in apple_nvme_free_prps()
358 static void apple_nvme_unmap_data(struct apple_nvme *anv, struct request *req) in apple_nvme_unmap_data() argument
360 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_unmap_data()
362 if (iod->dma_len) { in apple_nvme_unmap_data()
363 dma_unmap_page(anv->dev, iod->first_dma, iod->dma_len, in apple_nvme_unmap_data()
364 rq_dma_dir(req)); in apple_nvme_unmap_data()
368 WARN_ON_ONCE(!iod->nents); in apple_nvme_unmap_data()
370 dma_unmap_sg(anv->dev, iod->sg, iod->nents, rq_dma_dir(req)); in apple_nvme_unmap_data()
371 if (iod->npages == 0) in apple_nvme_unmap_data()
372 dma_pool_free(anv->prp_small_pool, apple_nvme_iod_list(req)[0], in apple_nvme_unmap_data()
373 iod->first_dma); in apple_nvme_unmap_data()
375 apple_nvme_free_prps(anv, req); in apple_nvme_unmap_data()
376 mempool_free(iod->sg, anv->iod_mempool); in apple_nvme_unmap_data()
388 i, &phys, sg->offset, sg->length, &sg_dma_address(sg), in apple_nvme_print_sgl()
394 struct request *req, in apple_nvme_setup_prps() argument
397 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_setup_prps()
399 int length = blk_rq_payload_bytes(req); in apple_nvme_setup_prps()
400 struct scatterlist *sg = iod->sg; in apple_nvme_setup_prps()
403 int offset = dma_addr & (NVME_CTRL_PAGE_SIZE - 1); in apple_nvme_setup_prps()
405 void **list = apple_nvme_iod_list(req); in apple_nvme_setup_prps()
409 length -= (NVME_CTRL_PAGE_SIZE - offset); in apple_nvme_setup_prps()
411 iod->first_dma = 0; in apple_nvme_setup_prps()
415 dma_len -= (NVME_CTRL_PAGE_SIZE - offset); in apple_nvme_setup_prps()
417 dma_addr += (NVME_CTRL_PAGE_SIZE - offset); in apple_nvme_setup_prps()
425 iod->first_dma = dma_addr; in apple_nvme_setup_prps()
431 pool = anv->prp_small_pool; in apple_nvme_setup_prps()
432 iod->npages = 0; in apple_nvme_setup_prps()
434 pool = anv->prp_page_pool; in apple_nvme_setup_prps()
435 iod->npages = 1; in apple_nvme_setup_prps()
440 iod->first_dma = dma_addr; in apple_nvme_setup_prps()
441 iod->npages = -1; in apple_nvme_setup_prps()
445 iod->first_dma = prp_dma; in apple_nvme_setup_prps()
454 list[iod->npages++] = prp_list; in apple_nvme_setup_prps()
455 prp_list[0] = old_prp_list[i - 1]; in apple_nvme_setup_prps()
456 old_prp_list[i - 1] = cpu_to_le64(prp_dma); in apple_nvme_setup_prps()
460 dma_len -= NVME_CTRL_PAGE_SIZE; in apple_nvme_setup_prps()
462 length -= NVME_CTRL_PAGE_SIZE; in apple_nvme_setup_prps()
474 cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg)); in apple_nvme_setup_prps()
475 cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma); in apple_nvme_setup_prps()
478 apple_nvme_free_prps(anv, req); in apple_nvme_setup_prps()
481 WARN(DO_ONCE(apple_nvme_print_sgl, iod->sg, iod->nents), in apple_nvme_setup_prps()
482 "Invalid SGL for payload:%d nents:%d\n", blk_rq_payload_bytes(req), in apple_nvme_setup_prps()
483 iod->nents); in apple_nvme_setup_prps()
488 struct request *req, in apple_nvme_setup_prp_simple() argument
492 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_setup_prp_simple()
493 unsigned int offset = bv->bv_offset & (NVME_CTRL_PAGE_SIZE - 1); in apple_nvme_setup_prp_simple()
494 unsigned int first_prp_len = NVME_CTRL_PAGE_SIZE - offset; in apple_nvme_setup_prp_simple()
496 iod->first_dma = dma_map_bvec(anv->dev, bv, rq_dma_dir(req), 0); in apple_nvme_setup_prp_simple()
497 if (dma_mapping_error(anv->dev, iod->first_dma)) in apple_nvme_setup_prp_simple()
499 iod->dma_len = bv->bv_len; in apple_nvme_setup_prp_simple()
501 cmnd->dptr.prp1 = cpu_to_le64(iod->first_dma); in apple_nvme_setup_prp_simple()
502 if (bv->bv_len > first_prp_len) in apple_nvme_setup_prp_simple()
503 cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma + first_prp_len); in apple_nvme_setup_prp_simple()
508 struct request *req, in apple_nvme_map_data() argument
511 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_map_data()
515 if (blk_rq_nr_phys_segments(req) == 1) { in apple_nvme_map_data()
516 struct bio_vec bv = req_bvec(req); in apple_nvme_map_data()
519 return apple_nvme_setup_prp_simple(anv, req, &cmnd->rw, in apple_nvme_map_data()
523 iod->dma_len = 0; in apple_nvme_map_data()
524 iod->sg = mempool_alloc(anv->iod_mempool, GFP_ATOMIC); in apple_nvme_map_data()
525 if (!iod->sg) in apple_nvme_map_data()
527 sg_init_table(iod->sg, blk_rq_nr_phys_segments(req)); in apple_nvme_map_data()
528 iod->nents = blk_rq_map_sg(req->q, req, iod->sg); in apple_nvme_map_data()
529 if (!iod->nents) in apple_nvme_map_data()
532 nr_mapped = dma_map_sg_attrs(anv->dev, iod->sg, iod->nents, in apple_nvme_map_data()
533 rq_dma_dir(req), DMA_ATTR_NO_WARN); in apple_nvme_map_data()
537 ret = apple_nvme_setup_prps(anv, req, &cmnd->rw); in apple_nvme_map_data()
543 dma_unmap_sg(anv->dev, iod->sg, iod->nents, rq_dma_dir(req)); in apple_nvme_map_data()
545 mempool_free(iod->sg, anv->iod_mempool); in apple_nvme_map_data()
549 static __always_inline void apple_nvme_unmap_rq(struct request *req) in apple_nvme_unmap_rq() argument
551 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_unmap_rq()
552 struct apple_nvme *anv = queue_to_apple_nvme(iod->q); in apple_nvme_unmap_rq()
554 if (blk_rq_nr_phys_segments(req)) in apple_nvme_unmap_rq()
555 apple_nvme_unmap_data(anv, req); in apple_nvme_unmap_rq()
558 static void apple_nvme_complete_rq(struct request *req) in apple_nvme_complete_rq() argument
560 apple_nvme_unmap_rq(req); in apple_nvme_complete_rq()
561 nvme_complete_rq(req); in apple_nvme_complete_rq()
571 struct nvme_completion *hcqe = &q->cqes[q->cq_head]; in apple_nvme_cqe_pending()
573 return (le16_to_cpu(READ_ONCE(hcqe->status)) & 1) == q->cq_phase; in apple_nvme_cqe_pending()
579 if (q->is_adminq) in apple_nvme_queue_tagset()
580 return anv->admin_tagset.tags[0]; in apple_nvme_queue_tagset()
582 return anv->tagset.tags[0]; in apple_nvme_queue_tagset()
589 struct nvme_completion *cqe = &q->cqes[idx]; in apple_nvme_handle_cqe()
590 __u16 command_id = READ_ONCE(cqe->command_id); in apple_nvme_handle_cqe()
591 struct request *req; in apple_nvme_handle_cqe() local
595 req = nvme_find_rq(apple_nvme_queue_tagset(anv, q), command_id); in apple_nvme_handle_cqe()
596 if (unlikely(!req)) { in apple_nvme_handle_cqe()
597 dev_warn(anv->dev, "invalid id %d completed", command_id); in apple_nvme_handle_cqe()
601 if (!nvme_try_complete_req(req, cqe->status, cqe->result) && in apple_nvme_handle_cqe()
602 !blk_mq_add_to_batch(req, iob, in apple_nvme_handle_cqe()
603 nvme_req(req)->status != NVME_SC_SUCCESS, in apple_nvme_handle_cqe()
605 apple_nvme_complete_rq(req); in apple_nvme_handle_cqe()
610 u32 tmp = q->cq_head + 1; in apple_nvme_update_cq_head()
613 q->cq_head = 0; in apple_nvme_update_cq_head()
614 q->cq_phase ^= 1; in apple_nvme_update_cq_head()
616 q->cq_head = tmp; in apple_nvme_update_cq_head()
629 * load-load control dependency between phase and the rest of in apple_nvme_poll_cq()
633 apple_nvme_handle_cqe(q, iob, q->cq_head); in apple_nvme_poll_cq()
638 writel(q->cq_head, q->cq_db); in apple_nvme_poll_cq()
648 if (!READ_ONCE(q->enabled) && !force) in apple_nvme_handle_cq()
665 spin_lock_irqsave(&anv->lock, flags); in apple_nvme_irq()
666 if (apple_nvme_handle_cq(&anv->ioq, false)) in apple_nvme_irq()
668 if (apple_nvme_handle_cq(&anv->adminq, false)) in apple_nvme_irq()
670 spin_unlock_irqrestore(&anv->lock, flags); in apple_nvme_irq()
686 c.create_cq.prp1 = cpu_to_le64(anv->ioq.cq_dma_addr); in apple_nvme_create_cq()
688 c.create_cq.qsize = cpu_to_le16(APPLE_ANS_MAX_QUEUE_DEPTH - 1); in apple_nvme_create_cq()
692 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); in apple_nvme_create_cq()
702 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); in apple_nvme_remove_cq()
714 c.create_sq.prp1 = cpu_to_le64(anv->ioq.sq_dma_addr); in apple_nvme_create_sq()
716 c.create_sq.qsize = cpu_to_le16(APPLE_ANS_MAX_QUEUE_DEPTH - 1); in apple_nvme_create_sq()
720 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); in apple_nvme_create_sq()
730 return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); in apple_nvme_remove_sq()
736 struct nvme_ns *ns = hctx->queue->queuedata; in apple_nvme_queue_rq()
737 struct apple_nvme_queue *q = hctx->driver_data; in apple_nvme_queue_rq()
739 struct request *req = bd->rq; in apple_nvme_queue_rq() local
740 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_queue_rq()
741 struct nvme_command *cmnd = &iod->cmd; in apple_nvme_queue_rq()
744 iod->npages = -1; in apple_nvme_queue_rq()
745 iod->nents = 0; in apple_nvme_queue_rq()
751 if (unlikely(!READ_ONCE(q->enabled))) in apple_nvme_queue_rq()
754 if (!nvme_check_ready(&anv->ctrl, req, true)) in apple_nvme_queue_rq()
755 return nvme_fail_nonready_command(&anv->ctrl, req); in apple_nvme_queue_rq()
757 ret = nvme_setup_cmd(ns, req); in apple_nvme_queue_rq()
761 if (blk_rq_nr_phys_segments(req)) { in apple_nvme_queue_rq()
762 ret = apple_nvme_map_data(anv, req, cmnd); in apple_nvme_queue_rq()
767 nvme_start_request(req); in apple_nvme_queue_rq()
772 nvme_cleanup_cmd(req); in apple_nvme_queue_rq()
779 hctx->driver_data = data; in apple_nvme_init_hctx()
784 struct request *req, unsigned int hctx_idx, in apple_nvme_init_request() argument
787 struct apple_nvme_queue *q = set->driver_data; in apple_nvme_init_request()
789 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_init_request()
790 struct nvme_request *nreq = nvme_req(req); in apple_nvme_init_request()
792 iod->q = q; in apple_nvme_init_request()
793 nreq->ctrl = &anv->ctrl; in apple_nvme_init_request()
794 nreq->cmd = &iod->cmd; in apple_nvme_init_request()
801 enum nvme_ctrl_state state = nvme_ctrl_state(&anv->ctrl); in apple_nvme_disable()
802 u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS); in apple_nvme_disable()
806 if (apple_rtkit_is_crashed(anv->rtk)) in apple_nvme_disable()
816 nvme_start_freeze(&anv->ctrl); in apple_nvme_disable()
824 nvme_wait_freeze_timeout(&anv->ctrl, NVME_IO_TIMEOUT); in apple_nvme_disable()
826 nvme_quiesce_io_queues(&anv->ctrl); in apple_nvme_disable()
829 if (READ_ONCE(anv->ioq.enabled)) { in apple_nvme_disable()
844 * Both U-Boot and m1n1 also use this convention (i.e. an ANS in apple_nvme_disable()
849 nvme_disable_ctrl(&anv->ctrl, shutdown); in apple_nvme_disable()
850 nvme_disable_ctrl(&anv->ctrl, false); in apple_nvme_disable()
853 WRITE_ONCE(anv->ioq.enabled, false); in apple_nvme_disable()
854 WRITE_ONCE(anv->adminq.enabled, false); in apple_nvme_disable()
856 nvme_quiesce_admin_queue(&anv->ctrl); in apple_nvme_disable()
859 spin_lock_irqsave(&anv->lock, flags); in apple_nvme_disable()
860 apple_nvme_handle_cq(&anv->ioq, true); in apple_nvme_disable()
861 apple_nvme_handle_cq(&anv->adminq, true); in apple_nvme_disable()
862 spin_unlock_irqrestore(&anv->lock, flags); in apple_nvme_disable()
864 nvme_cancel_tagset(&anv->ctrl); in apple_nvme_disable()
865 nvme_cancel_admin_tagset(&anv->ctrl); in apple_nvme_disable()
870 * deadlocking blk-mq hot-cpu notifier. in apple_nvme_disable()
873 nvme_unquiesce_io_queues(&anv->ctrl); in apple_nvme_disable()
874 nvme_unquiesce_admin_queue(&anv->ctrl); in apple_nvme_disable()
878 static enum blk_eh_timer_return apple_nvme_timeout(struct request *req) in apple_nvme_timeout() argument
880 struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req); in apple_nvme_timeout()
881 struct apple_nvme_queue *q = iod->q; in apple_nvme_timeout()
884 u32 csts = readl(anv->mmio_nvme + NVME_REG_CSTS); in apple_nvme_timeout()
886 if (nvme_ctrl_state(&anv->ctrl) != NVME_CTRL_LIVE) { in apple_nvme_timeout()
892 * - ctrl disable/shutdown fabrics requests in apple_nvme_timeout()
893 * - connect requests in apple_nvme_timeout()
894 * - initialization admin requests in apple_nvme_timeout()
895 * - I/O requests that entered after unquiescing and in apple_nvme_timeout()
901 dev_warn(anv->dev, in apple_nvme_timeout()
903 req->tag, q->is_adminq); in apple_nvme_timeout()
904 if (blk_mq_request_started(req) && in apple_nvme_timeout()
905 !blk_mq_request_completed(req)) { in apple_nvme_timeout()
906 nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD; in apple_nvme_timeout()
907 nvme_req(req)->flags |= NVME_REQ_CANCELLED; in apple_nvme_timeout()
908 blk_mq_complete_request(req); in apple_nvme_timeout()
914 if (!apple_rtkit_is_crashed(anv->rtk) && !(csts & NVME_CSTS_CFS)) { in apple_nvme_timeout()
915 spin_lock_irqsave(&anv->lock, flags); in apple_nvme_timeout()
917 spin_unlock_irqrestore(&anv->lock, flags); in apple_nvme_timeout()
918 if (blk_mq_request_completed(req)) { in apple_nvme_timeout()
919 dev_warn(anv->dev, in apple_nvme_timeout()
921 req->tag, q->is_adminq); in apple_nvme_timeout()
930 dev_warn(anv->dev, "I/O %d(aq:%d) timeout: resetting controller\n", in apple_nvme_timeout()
931 req->tag, q->is_adminq); in apple_nvme_timeout()
932 nvme_req(req)->flags |= NVME_REQ_CANCELLED; in apple_nvme_timeout()
934 nvme_reset_ctrl(&anv->ctrl); in apple_nvme_timeout()
941 struct apple_nvme_queue *q = hctx->driver_data; in apple_nvme_poll()
946 spin_lock_irqsave(&anv->lock, flags); in apple_nvme_poll()
948 spin_unlock_irqrestore(&anv->lock, flags); in apple_nvme_poll()
974 q->cq_head = 0; in apple_nvme_init_queue()
975 q->cq_phase = 1; in apple_nvme_init_queue()
976 memset(q->tcbs, 0, in apple_nvme_init_queue()
978 memset(q->cqes, 0, depth * sizeof(struct nvme_completion)); in apple_nvme_init_queue()
979 WRITE_ONCE(q->enabled, true); in apple_nvme_init_queue()
990 enum nvme_ctrl_state state = nvme_ctrl_state(&anv->ctrl); in apple_nvme_reset_work()
993 dev_warn(anv->dev, "ctrl state %d is not RESETTING\n", state); in apple_nvme_reset_work()
994 ret = -ENODEV; in apple_nvme_reset_work()
999 if (apple_rtkit_is_crashed(anv->rtk)) { in apple_nvme_reset_work()
1000 dev_err(anv->dev, in apple_nvme_reset_work()
1002 ret = -EIO; in apple_nvme_reset_work()
1006 /* RTKit must be shut down cleanly for the (soft)-reset to work */ in apple_nvme_reset_work()
1007 if (apple_rtkit_is_running(anv->rtk)) { in apple_nvme_reset_work()
1009 if (anv->ctrl.ctrl_config & NVME_CC_ENABLE) in apple_nvme_reset_work()
1011 dev_dbg(anv->dev, "Trying to shut down RTKit before reset."); in apple_nvme_reset_work()
1012 ret = apple_rtkit_shutdown(anv->rtk); in apple_nvme_reset_work()
1016 writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); in apple_nvme_reset_work()
1020 * Only do the soft-reset if the CPU is not running, which means either we in apple_nvme_reset_work()
1023 if (!(readl(anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL) & in apple_nvme_reset_work()
1026 ret = reset_control_assert(anv->reset); in apple_nvme_reset_work()
1030 ret = apple_rtkit_reinit(anv->rtk); in apple_nvme_reset_work()
1034 ret = reset_control_deassert(anv->reset); in apple_nvme_reset_work()
1039 anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); in apple_nvme_reset_work()
1041 ret = apple_rtkit_boot(anv->rtk); in apple_nvme_reset_work()
1043 ret = apple_rtkit_wake(anv->rtk); in apple_nvme_reset_work()
1047 dev_err(anv->dev, "ANS did not boot"); in apple_nvme_reset_work()
1051 ret = readl_poll_timeout(anv->mmio_nvme + APPLE_ANS_BOOT_STATUS, in apple_nvme_reset_work()
1056 dev_err(anv->dev, "ANS did not initialize"); in apple_nvme_reset_work()
1060 dev_dbg(anv->dev, "ANS booted successfully."); in apple_nvme_reset_work()
1063 * Limit the max command size to prevent iod->sg allocations going in apple_nvme_reset_work()
1066 anv->ctrl.max_hw_sectors = min_t(u32, NVME_MAX_KB_SZ << 1, in apple_nvme_reset_work()
1067 dma_max_mapping_size(anv->dev) >> 9); in apple_nvme_reset_work()
1068 anv->ctrl.max_segments = NVME_MAX_SEGS; in apple_nvme_reset_work()
1070 dma_set_max_seg_size(anv->dev, 0xffffffff); in apple_nvme_reset_work()
1080 anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL); in apple_nvme_reset_work()
1084 anv->mmio_nvme + APPLE_ANS_MAX_PEND_CMDS_CTRL); in apple_nvme_reset_work()
1087 writel(APPLE_ANS_MAX_QUEUE_DEPTH - 1, in apple_nvme_reset_work()
1088 anv->mmio_nvme + APPLE_NVMMU_NUM_TCBS); in apple_nvme_reset_work()
1093 * the co-processor complains about "completed with err BAD_CMD-" or in apple_nvme_reset_work()
1096 writel(readl(anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL) & in apple_nvme_reset_work()
1098 anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL); in apple_nvme_reset_work()
1101 aqa = APPLE_NVME_AQ_DEPTH - 1; in apple_nvme_reset_work()
1103 writel(aqa, anv->mmio_nvme + NVME_REG_AQA); in apple_nvme_reset_work()
1104 writeq(anv->adminq.sq_dma_addr, anv->mmio_nvme + NVME_REG_ASQ); in apple_nvme_reset_work()
1105 writeq(anv->adminq.cq_dma_addr, anv->mmio_nvme + NVME_REG_ACQ); in apple_nvme_reset_work()
1108 writeq(anv->adminq.tcb_dma_addr, in apple_nvme_reset_work()
1109 anv->mmio_nvme + APPLE_NVMMU_ASQ_TCB_BASE); in apple_nvme_reset_work()
1110 writeq(anv->ioq.tcb_dma_addr, in apple_nvme_reset_work()
1111 anv->mmio_nvme + APPLE_NVMMU_IOSQ_TCB_BASE); in apple_nvme_reset_work()
1113 anv->ctrl.sqsize = in apple_nvme_reset_work()
1114 APPLE_ANS_MAX_QUEUE_DEPTH - 1; /* 0's based queue depth */ in apple_nvme_reset_work()
1115 anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP); in apple_nvme_reset_work()
1117 dev_dbg(anv->dev, "Enabling controller now"); in apple_nvme_reset_work()
1118 ret = nvme_enable_ctrl(&anv->ctrl); in apple_nvme_reset_work()
1122 dev_dbg(anv->dev, "Starting admin queue"); in apple_nvme_reset_work()
1123 apple_nvme_init_queue(&anv->adminq); in apple_nvme_reset_work()
1124 nvme_unquiesce_admin_queue(&anv->ctrl); in apple_nvme_reset_work()
1126 if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_CONNECTING)) { in apple_nvme_reset_work()
1127 dev_warn(anv->ctrl.device, in apple_nvme_reset_work()
1129 ret = -ENODEV; in apple_nvme_reset_work()
1133 ret = nvme_init_ctrl_finish(&anv->ctrl, false); in apple_nvme_reset_work()
1137 dev_dbg(anv->dev, "Creating IOCQ"); in apple_nvme_reset_work()
1141 dev_dbg(anv->dev, "Creating IOSQ"); in apple_nvme_reset_work()
1146 apple_nvme_init_queue(&anv->ioq); in apple_nvme_reset_work()
1148 ret = nvme_set_queue_count(&anv->ctrl, &nr_io_queues); in apple_nvme_reset_work()
1152 ret = -ENXIO; in apple_nvme_reset_work()
1156 anv->ctrl.queue_count = nr_io_queues + 1; in apple_nvme_reset_work()
1158 nvme_unquiesce_io_queues(&anv->ctrl); in apple_nvme_reset_work()
1159 nvme_wait_freeze(&anv->ctrl); in apple_nvme_reset_work()
1160 blk_mq_update_nr_hw_queues(&anv->tagset, 1); in apple_nvme_reset_work()
1161 nvme_unfreeze(&anv->ctrl); in apple_nvme_reset_work()
1163 if (!nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_LIVE)) { in apple_nvme_reset_work()
1164 dev_warn(anv->ctrl.device, in apple_nvme_reset_work()
1166 ret = -ENODEV; in apple_nvme_reset_work()
1170 nvme_start_ctrl(&anv->ctrl); in apple_nvme_reset_work()
1172 dev_dbg(anv->dev, "ANS boot and NVMe init completed."); in apple_nvme_reset_work()
1180 dev_warn(anv->ctrl.device, "Reset failure status: %d\n", ret); in apple_nvme_reset_work()
1181 nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING); in apple_nvme_reset_work()
1182 nvme_get_ctrl(&anv->ctrl); in apple_nvme_reset_work()
1184 nvme_mark_namespaces_dead(&anv->ctrl); in apple_nvme_reset_work()
1185 if (!queue_work(nvme_wq, &anv->remove_work)) in apple_nvme_reset_work()
1186 nvme_put_ctrl(&anv->ctrl); in apple_nvme_reset_work()
1194 nvme_put_ctrl(&anv->ctrl); in apple_nvme_remove_dead_ctrl_work()
1195 device_release_driver(anv->dev); in apple_nvme_remove_dead_ctrl_work()
1200 *val = readl(ctrl_to_apple_nvme(ctrl)->mmio_nvme + off); in apple_nvme_reg_read32()
1206 writel(val, ctrl_to_apple_nvme(ctrl)->mmio_nvme + off); in apple_nvme_reg_write32()
1212 *val = readq(ctrl_to_apple_nvme(ctrl)->mmio_nvme + off); in apple_nvme_reg_read64()
1218 struct device *dev = ctrl_to_apple_nvme(ctrl)->dev; in apple_nvme_get_address()
1227 if (anv->ctrl.admin_q) in apple_nvme_free_ctrl()
1228 blk_put_queue(anv->ctrl.admin_q); in apple_nvme_free_ctrl()
1229 put_device(anv->dev); in apple_nvme_free_ctrl()
1233 .name = "apple-nvme",
1247 flush_work(&anv->ctrl.reset_work); in apple_nvme_async_probe()
1248 flush_work(&anv->ctrl.scan_work); in apple_nvme_async_probe()
1249 nvme_put_ctrl(&anv->ctrl); in apple_nvme_async_probe()
1261 anv->admin_tagset.ops = &apple_nvme_mq_admin_ops; in apple_nvme_alloc_tagsets()
1262 anv->admin_tagset.nr_hw_queues = 1; in apple_nvme_alloc_tagsets()
1263 anv->admin_tagset.queue_depth = APPLE_NVME_AQ_MQ_TAG_DEPTH; in apple_nvme_alloc_tagsets()
1264 anv->admin_tagset.timeout = NVME_ADMIN_TIMEOUT; in apple_nvme_alloc_tagsets()
1265 anv->admin_tagset.numa_node = NUMA_NO_NODE; in apple_nvme_alloc_tagsets()
1266 anv->admin_tagset.cmd_size = sizeof(struct apple_nvme_iod); in apple_nvme_alloc_tagsets()
1267 anv->admin_tagset.driver_data = &anv->adminq; in apple_nvme_alloc_tagsets()
1269 ret = blk_mq_alloc_tag_set(&anv->admin_tagset); in apple_nvme_alloc_tagsets()
1272 ret = devm_add_action_or_reset(anv->dev, devm_apple_nvme_put_tag_set, in apple_nvme_alloc_tagsets()
1273 &anv->admin_tagset); in apple_nvme_alloc_tagsets()
1277 anv->tagset.ops = &apple_nvme_mq_ops; in apple_nvme_alloc_tagsets()
1278 anv->tagset.nr_hw_queues = 1; in apple_nvme_alloc_tagsets()
1279 anv->tagset.nr_maps = 1; in apple_nvme_alloc_tagsets()
1285 anv->tagset.reserved_tags = APPLE_NVME_AQ_DEPTH; in apple_nvme_alloc_tagsets()
1286 anv->tagset.queue_depth = APPLE_ANS_MAX_QUEUE_DEPTH - 1; in apple_nvme_alloc_tagsets()
1287 anv->tagset.timeout = NVME_IO_TIMEOUT; in apple_nvme_alloc_tagsets()
1288 anv->tagset.numa_node = NUMA_NO_NODE; in apple_nvme_alloc_tagsets()
1289 anv->tagset.cmd_size = sizeof(struct apple_nvme_iod); in apple_nvme_alloc_tagsets()
1290 anv->tagset.driver_data = &anv->ioq; in apple_nvme_alloc_tagsets()
1292 ret = blk_mq_alloc_tag_set(&anv->tagset); in apple_nvme_alloc_tagsets()
1295 ret = devm_add_action_or_reset(anv->dev, devm_apple_nvme_put_tag_set, in apple_nvme_alloc_tagsets()
1296 &anv->tagset); in apple_nvme_alloc_tagsets()
1300 anv->ctrl.admin_tagset = &anv->admin_tagset; in apple_nvme_alloc_tagsets()
1301 anv->ctrl.tagset = &anv->tagset; in apple_nvme_alloc_tagsets()
1311 q->cqes = dmam_alloc_coherent(anv->dev, in apple_nvme_queue_alloc()
1313 &q->cq_dma_addr, GFP_KERNEL); in apple_nvme_queue_alloc()
1314 if (!q->cqes) in apple_nvme_queue_alloc()
1315 return -ENOMEM; in apple_nvme_queue_alloc()
1317 q->sqes = dmam_alloc_coherent(anv->dev, in apple_nvme_queue_alloc()
1319 &q->sq_dma_addr, GFP_KERNEL); in apple_nvme_queue_alloc()
1320 if (!q->sqes) in apple_nvme_queue_alloc()
1321 return -ENOMEM; in apple_nvme_queue_alloc()
1327 q->tcbs = dmam_alloc_coherent(anv->dev, in apple_nvme_queue_alloc()
1330 &q->tcb_dma_addr, GFP_KERNEL); in apple_nvme_queue_alloc()
1331 if (!q->tcbs) in apple_nvme_queue_alloc()
1332 return -ENOMEM; in apple_nvme_queue_alloc()
1338 q->cq_phase = 1; in apple_nvme_queue_alloc()
1346 if (anv->pd_count <= 1) in apple_nvme_detach_genpd()
1349 for (i = anv->pd_count - 1; i >= 0; i--) { in apple_nvme_detach_genpd()
1350 if (anv->pd_link[i]) in apple_nvme_detach_genpd()
1351 device_link_del(anv->pd_link[i]); in apple_nvme_detach_genpd()
1352 if (!IS_ERR_OR_NULL(anv->pd_dev[i])) in apple_nvme_detach_genpd()
1353 dev_pm_domain_detach(anv->pd_dev[i], true); in apple_nvme_detach_genpd()
1359 struct device *dev = anv->dev; in apple_nvme_attach_genpd()
1362 anv->pd_count = of_count_phandle_with_args( in apple_nvme_attach_genpd()
1363 dev->of_node, "power-domains", "#power-domain-cells"); in apple_nvme_attach_genpd()
1364 if (anv->pd_count <= 1) in apple_nvme_attach_genpd()
1367 anv->pd_dev = devm_kcalloc(dev, anv->pd_count, sizeof(*anv->pd_dev), in apple_nvme_attach_genpd()
1369 if (!anv->pd_dev) in apple_nvme_attach_genpd()
1370 return -ENOMEM; in apple_nvme_attach_genpd()
1372 anv->pd_link = devm_kcalloc(dev, anv->pd_count, sizeof(*anv->pd_link), in apple_nvme_attach_genpd()
1374 if (!anv->pd_link) in apple_nvme_attach_genpd()
1375 return -ENOMEM; in apple_nvme_attach_genpd()
1377 for (i = 0; i < anv->pd_count; i++) { in apple_nvme_attach_genpd()
1378 anv->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i); in apple_nvme_attach_genpd()
1379 if (IS_ERR(anv->pd_dev[i])) { in apple_nvme_attach_genpd()
1381 return PTR_ERR(anv->pd_dev[i]); in apple_nvme_attach_genpd()
1384 anv->pd_link[i] = device_link_add(dev, anv->pd_dev[i], in apple_nvme_attach_genpd()
1388 if (!anv->pd_link[i]) { in apple_nvme_attach_genpd()
1390 return -EINVAL; in apple_nvme_attach_genpd()
1404 struct device *dev = &pdev->dev; in apple_nvme_alloc()
1410 return ERR_PTR(-ENOMEM); in apple_nvme_alloc()
1412 anv->dev = get_device(dev); in apple_nvme_alloc()
1413 anv->adminq.is_adminq = true; in apple_nvme_alloc()
1418 dev_err_probe(dev, ret, "Failed to attach power domains"); in apple_nvme_alloc()
1422 ret = -ENXIO; in apple_nvme_alloc()
1426 anv->irq = platform_get_irq(pdev, 0); in apple_nvme_alloc()
1427 if (anv->irq < 0) { in apple_nvme_alloc()
1428 ret = anv->irq; in apple_nvme_alloc()
1431 if (!anv->irq) { in apple_nvme_alloc()
1432 ret = -ENXIO; in apple_nvme_alloc()
1436 anv->mmio_coproc = devm_platform_ioremap_resource_byname(pdev, "ans"); in apple_nvme_alloc()
1437 if (IS_ERR(anv->mmio_coproc)) { in apple_nvme_alloc()
1438 ret = PTR_ERR(anv->mmio_coproc); in apple_nvme_alloc()
1441 anv->mmio_nvme = devm_platform_ioremap_resource_byname(pdev, "nvme"); in apple_nvme_alloc()
1442 if (IS_ERR(anv->mmio_nvme)) { in apple_nvme_alloc()
1443 ret = PTR_ERR(anv->mmio_nvme); in apple_nvme_alloc()
1447 anv->adminq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_ASQ_DB; in apple_nvme_alloc()
1448 anv->adminq.cq_db = anv->mmio_nvme + APPLE_ANS_ACQ_DB; in apple_nvme_alloc()
1449 anv->ioq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_IOSQ_DB; in apple_nvme_alloc()
1450 anv->ioq.cq_db = anv->mmio_nvme + APPLE_ANS_IOCQ_DB; in apple_nvme_alloc()
1452 anv->sart = devm_apple_sart_get(dev); in apple_nvme_alloc()
1453 if (IS_ERR(anv->sart)) { in apple_nvme_alloc()
1454 ret = dev_err_probe(dev, PTR_ERR(anv->sart), in apple_nvme_alloc()
1459 anv->reset = devm_reset_control_array_get_exclusive(anv->dev); in apple_nvme_alloc()
1460 if (IS_ERR(anv->reset)) { in apple_nvme_alloc()
1461 ret = dev_err_probe(dev, PTR_ERR(anv->reset), in apple_nvme_alloc()
1466 INIT_WORK(&anv->ctrl.reset_work, apple_nvme_reset_work); in apple_nvme_alloc()
1467 INIT_WORK(&anv->remove_work, apple_nvme_remove_dead_ctrl_work); in apple_nvme_alloc()
1468 spin_lock_init(&anv->lock); in apple_nvme_alloc()
1470 ret = apple_nvme_queue_alloc(anv, &anv->adminq); in apple_nvme_alloc()
1473 ret = apple_nvme_queue_alloc(anv, &anv->ioq); in apple_nvme_alloc()
1477 anv->prp_page_pool = dmam_pool_create("prp list page", anv->dev, in apple_nvme_alloc()
1480 if (!anv->prp_page_pool) { in apple_nvme_alloc()
1481 ret = -ENOMEM; in apple_nvme_alloc()
1485 anv->prp_small_pool = in apple_nvme_alloc()
1486 dmam_pool_create("prp list 256", anv->dev, 256, 256, 0); in apple_nvme_alloc()
1487 if (!anv->prp_small_pool) { in apple_nvme_alloc()
1488 ret = -ENOMEM; in apple_nvme_alloc()
1493 anv->iod_mempool = in apple_nvme_alloc()
1495 if (!anv->iod_mempool) { in apple_nvme_alloc()
1496 ret = -ENOMEM; in apple_nvme_alloc()
1499 ret = devm_add_action_or_reset(anv->dev, in apple_nvme_alloc()
1500 devm_apple_nvme_mempool_destroy, anv->iod_mempool); in apple_nvme_alloc()
1508 ret = devm_request_irq(anv->dev, anv->irq, apple_nvme_irq, 0, in apple_nvme_alloc()
1509 "nvme-apple", anv); in apple_nvme_alloc()
1515 anv->rtk = in apple_nvme_alloc()
1517 if (IS_ERR(anv->rtk)) { in apple_nvme_alloc()
1518 ret = dev_err_probe(dev, PTR_ERR(anv->rtk), in apple_nvme_alloc()
1523 ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops, in apple_nvme_alloc()
1533 put_device(anv->dev); in apple_nvme_alloc()
1546 ret = nvme_add_ctrl(&anv->ctrl); in apple_nvme_probe()
1550 anv->ctrl.admin_q = blk_mq_alloc_queue(&anv->admin_tagset, NULL, NULL); in apple_nvme_probe()
1551 if (IS_ERR(anv->ctrl.admin_q)) { in apple_nvme_probe()
1552 ret = -ENOMEM; in apple_nvme_probe()
1553 anv->ctrl.admin_q = NULL; in apple_nvme_probe()
1557 nvme_reset_ctrl(&anv->ctrl); in apple_nvme_probe()
1563 nvme_uninit_ctrl(&anv->ctrl); in apple_nvme_probe()
1565 nvme_put_ctrl(&anv->ctrl); in apple_nvme_probe()
1574 nvme_change_ctrl_state(&anv->ctrl, NVME_CTRL_DELETING); in apple_nvme_remove()
1575 flush_work(&anv->ctrl.reset_work); in apple_nvme_remove()
1576 nvme_stop_ctrl(&anv->ctrl); in apple_nvme_remove()
1577 nvme_remove_namespaces(&anv->ctrl); in apple_nvme_remove()
1579 nvme_uninit_ctrl(&anv->ctrl); in apple_nvme_remove()
1581 if (apple_rtkit_is_running(anv->rtk)) { in apple_nvme_remove()
1582 apple_rtkit_shutdown(anv->rtk); in apple_nvme_remove()
1584 writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); in apple_nvme_remove()
1595 if (apple_rtkit_is_running(anv->rtk)) { in apple_nvme_shutdown()
1596 apple_rtkit_shutdown(anv->rtk); in apple_nvme_shutdown()
1598 writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); in apple_nvme_shutdown()
1606 return nvme_reset_ctrl(&anv->ctrl); in apple_nvme_resume()
1616 if (apple_rtkit_is_running(anv->rtk)) { in apple_nvme_suspend()
1617 ret = apple_rtkit_shutdown(anv->rtk); in apple_nvme_suspend()
1619 writel(0, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); in apple_nvme_suspend()
1629 { .compatible = "apple,nvme-ans2" },
1636 .name = "nvme-apple",