1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
3  * Copyright (c) 2014-2017 Oracle.  All rights reserved.
4  * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
9  * COPYING in the main directory of this source tree, or the BSD-type
39  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45  * This file contains the top-level implementation of an RPC RDMA
49  * transport switch. All others are RPC RDMA internal.
74 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
82 static unsigned int max_memreg = RPCRDMA_LAST - 1;
152 	snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr));  in xprt_rdma_format_addresses4()
153 	xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL);  in xprt_rdma_format_addresses4()
155 	xprt->address_strings[RPC_DISPLAY_NETID] = RPCBIND_NETID_RDMA;  in xprt_rdma_format_addresses4()
164 	snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr);  in xprt_rdma_format_addresses6()
165 	xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL);  in xprt_rdma_format_addresses6()
167 	xprt->address_strings[RPC_DISPLAY_NETID] = RPCBIND_NETID_RDMA6;  in xprt_rdma_format_addresses6()
175 	switch (sap->sa_family) {  in xprt_rdma_format_addresses()
188 	xprt->address_strings[RPC_DISPLAY_ADDR] = kstrdup(buf, GFP_KERNEL);  in xprt_rdma_format_addresses()
191 	xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL);  in xprt_rdma_format_addresses()
194 	xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL);  in xprt_rdma_format_addresses()
196 	xprt->address_strings[RPC_DISPLAY_PROTO] = "rdma";  in xprt_rdma_format_addresses()
210 			kfree(xprt->address_strings[i]);  in xprt_rdma_free_addresses()
215  * xprt_rdma_connect_worker - establish connection in the background
219  * transport while a fresh connection is being established. RPC tasks
227 	struct rpc_xprt *xprt = &r_xprt->rx_xprt;  in xprt_rdma_connect_worker()
228 	unsigned int pflags = current->flags;  in xprt_rdma_connect_worker()
231 	if (atomic_read(&xprt->swapper))  in xprt_rdma_connect_worker()
232 		current->flags |= PF_MEMALLOC;  in xprt_rdma_connect_worker()
235 	if (!rc) {  in xprt_rdma_connect_worker()
236 		xprt->connect_cookie++;  in xprt_rdma_connect_worker()
237 		xprt->stat.connect_count++;  in xprt_rdma_connect_worker()
238 		xprt->stat.connect_time += (long)jiffies -  in xprt_rdma_connect_worker()
239 					   xprt->stat.connect_start;  in xprt_rdma_connect_worker()
241 		rc = -EAGAIN;  in xprt_rdma_connect_worker()
250  * xprt_rdma_inject_disconnect - inject a connection fault
253  * If @xprt is connected, disconnect it to simulate spurious
264 	rdma_disconnect(r_xprt->rx_ep->re_id);  in xprt_rdma_inject_disconnect()
268  * xprt_rdma_destroy - Full tear down of transport
279 	cancel_delayed_work_sync(&r_xprt->rx_connect_worker);  in xprt_rdma_destroy()
282 	rpcrdma_buffer_destroy(&r_xprt->rx_buf);  in xprt_rdma_destroy()
297  * xprt_setup_rdma - Set up transport to use RDMA
299  * @args: rpc transport arguments
309 	if (args->addrlen > sizeof(xprt->addr))  in xprt_setup_rdma()
310 		return ERR_PTR(-EBADF);  in xprt_setup_rdma()
312 	if (!try_module_get(THIS_MODULE))  in xprt_setup_rdma()
313 		return ERR_PTR(-EIO);  in xprt_setup_rdma()
315 	xprt = xprt_alloc(args->net, sizeof(struct rpcrdma_xprt), 0,  in xprt_setup_rdma()
317 	if (!xprt) {  in xprt_setup_rdma()
319 		return ERR_PTR(-ENOMEM);  in xprt_setup_rdma()
322 	xprt->timeout = &xprt_rdma_default_timeout;  in xprt_setup_rdma()
323 	xprt->connect_timeout = xprt->timeout->to_initval;  in xprt_setup_rdma()
324 	xprt->max_reconnect_timeout = xprt->timeout->to_maxval;  in xprt_setup_rdma()
325 	xprt->bind_timeout = RPCRDMA_BIND_TO;  in xprt_setup_rdma()
326 	xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO;  in xprt_setup_rdma()
327 	xprt->idle_timeout = RPCRDMA_IDLE_DISC_TO;  in xprt_setup_rdma()
329 	xprt->resvport = 0;		/* privileged port not needed */  in xprt_setup_rdma()
330 	xprt->ops = &xprt_rdma_procs;  in xprt_setup_rdma()
333 	 * Set up RDMA-specific connect data.  in xprt_setup_rdma()
335 	sap = args->dstaddr;  in xprt_setup_rdma()
337 	/* Ensure xprt->addr holds valid server TCP (not RDMA)  in xprt_setup_rdma()
339 	xprt->prot = IPPROTO_TCP;  in xprt_setup_rdma()
340 	xprt->xprt_class = &xprt_rdma;  in xprt_setup_rdma()
341 	xprt->addrlen = args->addrlen;  in xprt_setup_rdma()
342 	memcpy(&xprt->addr, sap, xprt->addrlen);  in xprt_setup_rdma()
344 	if (rpc_get_port(sap))  in xprt_setup_rdma()
350 	if (rc) {  in xprt_setup_rdma()
357 	INIT_DELAYED_WORK(&new_xprt->rx_connect_worker,  in xprt_setup_rdma()
360 	xprt->max_payload = RPCRDMA_MAX_DATA_SEGS << PAGE_SHIFT;  in xprt_setup_rdma()
366  * xprt_rdma_close - close a transport connection
380 	xprt->reestablish_timeout = 0;  in xprt_rdma_close()
381 	++xprt->connect_cookie;  in xprt_rdma_close()
386  * xprt_rdma_set_port - update server port with rpcbind result
387  * @xprt: controlling RPC transport
395 	struct sockaddr *sap = (struct sockaddr *)&xprt->addr;  in xprt_rdma_set_port()
400 	kfree(xprt->address_strings[RPC_DISPLAY_PORT]);  in xprt_rdma_set_port()
402 	xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL);  in xprt_rdma_set_port()
404 	kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]);  in xprt_rdma_set_port()
406 	xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL);  in xprt_rdma_set_port()
410  * xprt_rdma_timer - invoked when an RPC times out
411  * @xprt: controlling RPC transport
412  * @task: RPC task that timed out
414  * Invoked when the transport is still connected, but an RPC
417  * Since RDMA connections don't have a keep-alive, forcibly
429  * xprt_rdma_set_connect_timeout - set timeouts for establishing a connection
443 	spin_lock(&xprt->transport_lock);  in xprt_rdma_set_connect_timeout()
445 	if (connect_timeout < xprt->connect_timeout) {  in xprt_rdma_set_connect_timeout()
449 		to = *xprt->timeout;  in xprt_rdma_set_connect_timeout()
451 		if (initval < RPCRDMA_INIT_REEST_TO << 1)  in xprt_rdma_set_connect_timeout()
455 		r_xprt->rx_timeout = to;  in xprt_rdma_set_connect_timeout()
456 		xprt->timeout = &r_xprt->rx_timeout;  in xprt_rdma_set_connect_timeout()
457 		xprt->connect_timeout = connect_timeout;  in xprt_rdma_set_connect_timeout()
460 	if (reconnect_timeout < xprt->max_reconnect_timeout)  in xprt_rdma_set_connect_timeout()
461 		xprt->max_reconnect_timeout = reconnect_timeout;  in xprt_rdma_set_connect_timeout()
463 	spin_unlock(&xprt->transport_lock);  in xprt_rdma_set_connect_timeout()
467  * xprt_rdma_connect - schedule an attempt to reconnect
469  * @task: RPC scheduler context (unused)
476 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in xprt_rdma_connect()
482 	if (ep && ep->re_connect_status != 0) {  in xprt_rdma_connect()
487 	queue_delayed_work(system_long_wq, &r_xprt->rx_connect_worker, delay);  in xprt_rdma_connect()
491  * xprt_rdma_alloc_slot - allocate an rpc_rqst
492  * @xprt: controlling RPC transport
493  * @task: RPC task requesting a fresh rpc_rqst
496  *	%0 if task->tk_rqstp points to a fresh rpc_rqst
497  *	%-EAGAIN if no rpc_rqst is available; queued on backlog
505 	req = rpcrdma_buffer_get(&r_xprt->rx_buf);  in xprt_rdma_alloc_slot()
506 	if (!req)  in xprt_rdma_alloc_slot()
508 	task->tk_rqstp = &req->rl_slot;  in xprt_rdma_alloc_slot()
509 	task->tk_status = 0;  in xprt_rdma_alloc_slot()
513 	task->tk_status = -ENOMEM;  in xprt_rdma_alloc_slot()
518  * xprt_rdma_free_slot - release an rpc_rqst
519  * @xprt: controlling RPC transport
529 	rpcrdma_reply_put(&r_xprt->rx_buf, rpcr_to_rdmar(rqst));  in xprt_rdma_free_slot()
530 	if (!xprt_wake_up_backlog(xprt, rqst)) {  in xprt_rdma_free_slot()
532 		rpcrdma_buffer_put(&r_xprt->rx_buf, rpcr_to_rdmar(rqst));  in xprt_rdma_free_slot()
540 	if (unlikely(rdmab_length(rb) < size)) {  in rpcrdma_check_regbuf()
541 		if (!rpcrdma_regbuf_realloc(rb, size, flags))  in rpcrdma_check_regbuf()
543 		r_xprt->rx_stats.hardway_register_count += size;  in rpcrdma_check_regbuf()
549  * xprt_rdma_allocate - allocate transport resources for an RPC
550  * @task: RPC task
553  *        0:	Success; rq_buffer points to RPC buffer to use
560 	struct rpc_rqst *rqst = task->tk_rqstp;  in xprt_rdma_allocate()
561 	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt);  in xprt_rdma_allocate()
565 	if (!rpcrdma_check_regbuf(r_xprt, req->rl_sendbuf, rqst->rq_callsize,  in xprt_rdma_allocate()
568 	if (!rpcrdma_check_regbuf(r_xprt, req->rl_recvbuf, rqst->rq_rcvsize,  in xprt_rdma_allocate()
572 	rqst->rq_buffer = rdmab_data(req->rl_sendbuf);  in xprt_rdma_allocate()
573 	rqst->rq_rbuffer = rdmab_data(req->rl_recvbuf);  in xprt_rdma_allocate()
577 	return -ENOMEM;  in xprt_rdma_allocate()
581  * xprt_rdma_free - release resources allocated by xprt_rdma_allocate
582  * @task: RPC task
584  * Caller guarantees rqst->rq_buffer is non-NULL.
589 	struct rpc_rqst *rqst = task->tk_rqstp;  in xprt_rdma_free()
592 	if (unlikely(!list_empty(&req->rl_registered))) {  in xprt_rdma_free()
594 		frwr_unmap_sync(rpcx_to_rdmax(rqst->rq_xprt), req);  in xprt_rdma_free()
597 	/* XXX: If the RPC is completing because of a signal and  in xprt_rdma_free()
605  * xprt_rdma_send_request - marshal and send an RPC request
606  * @rqst: RPC message in rq_snd_buf
611  *	%0 if the RPC message has been sent
612  *	%-ENOTCONN if the caller should reconnect and call again
613  *	%-EAGAIN if the caller should call again
614  *	%-ENOBUFS if the caller should call again after a delay
615  *	%-EMSGSIZE if encoding ran out of buffer space. The request
617  *	%-EIO if an I/O error occurred. The request was not sent.
623 	struct rpc_xprt *xprt = rqst->rq_xprt;  in xprt_rdma_send_request()
628 #if defined(CONFIG_SUNRPC_BACKCHANNEL)  in xprt_rdma_send_request()
629 	if (unlikely(!rqst->rq_buffer))  in xprt_rdma_send_request()
633 	if (!xprt_connected(xprt))  in xprt_rdma_send_request()
634 		return -ENOTCONN;  in xprt_rdma_send_request()
636 	if (!xprt_request_get_cong(xprt, rqst))  in xprt_rdma_send_request()
637 		return -EBADSLT;  in xprt_rdma_send_request()
640 	if (rc < 0)  in xprt_rdma_send_request()
644 	if (rqst->rq_connect_cookie == xprt->connect_cookie)  in xprt_rdma_send_request()
646 	rqst->rq_xtime = ktime_get();  in xprt_rdma_send_request()
648 	if (frwr_send(r_xprt, req))  in xprt_rdma_send_request()
651 	rqst->rq_xmit_bytes_sent += rqst->rq_snd_buf.len;  in xprt_rdma_send_request()
653 	/* An RPC with no reply will throw off credit accounting,  in xprt_rdma_send_request()
656 	if (!rpc_reply_expected(rqst->rq_task))  in xprt_rdma_send_request()
661 	if (rc != -ENOTCONN)  in xprt_rdma_send_request()
665 	return -ENOTCONN;  in xprt_rdma_send_request()
673 	if (xprt_connected(xprt))  in xprt_rdma_print_stats()
674 		idle_time = (long)(jiffies - xprt->last_used) / HZ;  in xprt_rdma_print_stats()
679 		   xprt->stat.bind_count,  in xprt_rdma_print_stats()
680 		   xprt->stat.connect_count,  in xprt_rdma_print_stats()
681 		   xprt->stat.connect_time / HZ,  in xprt_rdma_print_stats()
683 		   xprt->stat.sends,  in xprt_rdma_print_stats()
684 		   xprt->stat.recvs,  in xprt_rdma_print_stats()
685 		   xprt->stat.bad_xids,  in xprt_rdma_print_stats()
686 		   xprt->stat.req_u,  in xprt_rdma_print_stats()
687 		   xprt->stat.bklog_u);  in xprt_rdma_print_stats()
689 		   r_xprt->rx_stats.read_chunk_count,  in xprt_rdma_print_stats()
690 		   r_xprt->rx_stats.write_chunk_count,  in xprt_rdma_print_stats()
691 		   r_xprt->rx_stats.reply_chunk_count,  in xprt_rdma_print_stats()
692 		   r_xprt->rx_stats.total_rdma_request,  in xprt_rdma_print_stats()
693 		   r_xprt->rx_stats.total_rdma_reply,  in xprt_rdma_print_stats()
694 		   r_xprt->rx_stats.pullup_copy_count,  in xprt_rdma_print_stats()
695 		   r_xprt->rx_stats.fixup_copy_count,  in xprt_rdma_print_stats()
696 		   r_xprt->rx_stats.hardway_register_count,  in xprt_rdma_print_stats()
697 		   r_xprt->rx_stats.failed_marshal_count,  in xprt_rdma_print_stats()
698 		   r_xprt->rx_stats.bad_reply_count,  in xprt_rdma_print_stats()
699 		   r_xprt->rx_stats.nomsg_call_count);  in xprt_rdma_print_stats()
701 		   r_xprt->rx_stats.mrs_recycled,  in xprt_rdma_print_stats()
702 		   r_xprt->rx_stats.mrs_orphaned,  in xprt_rdma_print_stats()
703 		   r_xprt->rx_stats.mrs_allocated,  in xprt_rdma_print_stats()
704 		   r_xprt->rx_stats.local_inv_needed,  in xprt_rdma_print_stats()
705 		   r_xprt->rx_stats.empty_sendctx_q,  in xprt_rdma_print_stats()
706 		   r_xprt->rx_stats.reply_waits_for_send);  in xprt_rdma_print_stats()
721  * Plumbing for rpc transport switch and kernel module
745 #if defined(CONFIG_SUNRPC_BACKCHANNEL)
765 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)  in xprt_rdma_cleanup()
766 	if (sunrpc_table_header) {  in xprt_rdma_cleanup()
781 	if (rc)  in xprt_rdma_init()
785 	if (rc) {  in xprt_rdma_init()
790 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)  in xprt_rdma_init()
791 	if (!sunrpc_table_header)  in xprt_rdma_init()