31 	struct smc_connection *conn = cdcpend->conn;  in smc_cdc_tx_handler()  local
36 	sndbuf_desc = conn->sndbuf_desc;  in smc_cdc_tx_handler()
37 	smc = container_of(conn, struct smc_sock, conn);  in smc_cdc_tx_handler()
41 				     &cdcpend->conn->tx_curs_fin,  in smc_cdc_tx_handler()
45 		atomic_add(diff, &cdcpend->conn->sndbuf_space);  in smc_cdc_tx_handler()
48 		smc_curs_copy(&conn->tx_curs_fin, &cdcpend->cursor, conn);  in smc_cdc_tx_handler()
49 		smc_curs_copy(&conn->local_tx_ctrl_fin, &cdcpend->p_cursor,  in smc_cdc_tx_handler()
50 			      conn);  in smc_cdc_tx_handler()
51 		conn->tx_cdc_seq_fin = cdcpend->ctrl_seq;  in smc_cdc_tx_handler()
54 	if (atomic_dec_and_test(&conn->cdc_pend_tx_wr)) {  in smc_cdc_tx_handler()
60 			conn->tx_in_release_sock = true;  in smc_cdc_tx_handler()
62 			smc_tx_pending(conn);  in smc_cdc_tx_handler()
64 		if (unlikely(wq_has_sleeper(&conn->cdc_pend_tx_wq)))  in smc_cdc_tx_handler()
65 			wake_up(&conn->cdc_pend_tx_wq);  in smc_cdc_tx_handler()
67 	WARN_ON(atomic_read(&conn->cdc_pend_tx_wr) < 0);  in smc_cdc_tx_handler()
73 int smc_cdc_get_free_slot(struct smc_connection *conn,  in smc_cdc_get_free_slot()  argument
84 	if (conn->killed) {  in smc_cdc_get_free_slot()
94 static inline void smc_cdc_add_pending_send(struct smc_connection *conn,  in smc_cdc_add_pending_send()  argument
106 	pend->conn = conn;  in smc_cdc_add_pending_send()
107 	pend->cursor = conn->tx_curs_sent;  in smc_cdc_add_pending_send()
108 	pend->p_cursor = conn->local_tx_ctrl.prod;  in smc_cdc_add_pending_send()
109 	pend->ctrl_seq = conn->tx_cdc_seq;  in smc_cdc_add_pending_send()
112 int smc_cdc_msg_send(struct smc_connection *conn,  in smc_cdc_msg_send()  argument
116 	struct smc_link *link = conn->lnk;  in smc_cdc_msg_send()
120 	smc_cdc_add_pending_send(conn, pend);  in smc_cdc_msg_send()
122 	conn->tx_cdc_seq++;  in smc_cdc_msg_send()
123 	conn->local_tx_ctrl.seqno = conn->tx_cdc_seq;  in smc_cdc_msg_send()
124 	smc_host_msg_to_cdc((struct smc_cdc_msg *)wr_buf, conn, &cfed);  in smc_cdc_msg_send()
126 	atomic_inc(&conn->cdc_pend_tx_wr);  in smc_cdc_msg_send()
131 		smc_curs_copy(&conn->rx_curs_confirmed, &cfed, conn);  in smc_cdc_msg_send()
132 		conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0;  in smc_cdc_msg_send()
134 		conn->tx_cdc_seq--;  in smc_cdc_msg_send()
135 		conn->local_tx_ctrl.seqno = conn->tx_cdc_seq;  in smc_cdc_msg_send()
136 		atomic_dec(&conn->cdc_pend_tx_wr);  in smc_cdc_msg_send()
142 /* send a validation msg indicating the move of a conn to an other QP link */
143 int smcr_cdc_msg_send_validation(struct smc_connection *conn,  in smcr_cdc_msg_send_validation()  argument
147 	struct smc_host_cdc_msg *local = &conn->local_tx_ctrl;  in smcr_cdc_msg_send_validation()
148 	struct smc_link *link = conn->lnk;  in smcr_cdc_msg_send_validation()
155 	peer->seqno = htons(conn->tx_cdc_seq_fin); /* seqno last compl. tx */  in smcr_cdc_msg_send_validation()
159 	/* We need to set pend->conn here to make sure smc_cdc_tx_handler()  in smcr_cdc_msg_send_validation()
162 	smc_cdc_add_pending_send(conn, pend);  in smcr_cdc_msg_send_validation()
164 	atomic_inc(&conn->cdc_pend_tx_wr);  in smcr_cdc_msg_send_validation()
169 		atomic_dec(&conn->cdc_pend_tx_wr);  in smcr_cdc_msg_send_validation()
174 static int smcr_cdc_get_slot_and_msg_send(struct smc_connection *conn)  in smcr_cdc_get_slot_and_msg_send()  argument
183 	link = conn->lnk;  in smcr_cdc_get_slot_and_msg_send()
186 	rc = smc_cdc_get_free_slot(conn, link, &wr_buf, NULL, &pend);  in smcr_cdc_get_slot_and_msg_send()
190 	spin_lock_bh(&conn->send_lock);  in smcr_cdc_get_slot_and_msg_send()
191 	if (link != conn->lnk) {  in smcr_cdc_get_slot_and_msg_send()
193 		spin_unlock_bh(&conn->send_lock);  in smcr_cdc_get_slot_and_msg_send()
202 	rc = smc_cdc_msg_send(conn, wr_buf, pend);  in smcr_cdc_get_slot_and_msg_send()
203 	spin_unlock_bh(&conn->send_lock);  in smcr_cdc_get_slot_and_msg_send()
209 int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn)  in smc_cdc_get_slot_and_msg_send()  argument
213 	if (!smc_conn_lgr_valid(conn) ||  in smc_cdc_get_slot_and_msg_send()
214 	    (conn->lgr->is_smcd && conn->lgr->peer_shutdown))  in smc_cdc_get_slot_and_msg_send()
217 	if (conn->lgr->is_smcd) {  in smc_cdc_get_slot_and_msg_send()
218 		spin_lock_bh(&conn->send_lock);  in smc_cdc_get_slot_and_msg_send()
219 		rc = smcd_cdc_msg_send(conn);  in smc_cdc_get_slot_and_msg_send()
220 		spin_unlock_bh(&conn->send_lock);  in smc_cdc_get_slot_and_msg_send()
222 		rc = smcr_cdc_get_slot_and_msg_send(conn);  in smc_cdc_get_slot_and_msg_send()
228 void smc_cdc_wait_pend_tx_wr(struct smc_connection *conn)  in smc_cdc_wait_pend_tx_wr()  argument
230 	wait_event(conn->cdc_pend_tx_wq, !atomic_read(&conn->cdc_pend_tx_wr));  in smc_cdc_wait_pend_tx_wr()
237 int smcd_cdc_msg_send(struct smc_connection *conn)  in smcd_cdc_msg_send()  argument
239 	struct smc_sock *smc = container_of(conn, struct smc_sock, conn);  in smcd_cdc_msg_send()
246 	curs.acurs.counter = atomic64_read(&conn->local_tx_ctrl.prod.acurs);  in smcd_cdc_msg_send()
249 	curs.acurs.counter = atomic64_read(&conn->local_tx_ctrl.cons.acurs);  in smcd_cdc_msg_send()
252 	cdc.cons.prod_flags = conn->local_tx_ctrl.prod_flags;  in smcd_cdc_msg_send()
253 	cdc.cons.conn_state_flags = conn->local_tx_ctrl.conn_state_flags;  in smcd_cdc_msg_send()
254 	rc = smcd_tx_ism_write(conn, &cdc, sizeof(cdc), 0, 1);  in smcd_cdc_msg_send()
257 	smc_curs_copy(&conn->rx_curs_confirmed, &curs, conn);  in smcd_cdc_msg_send()
258 	conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0;  in smcd_cdc_msg_send()
260 	if (smc_ism_support_dmb_nocopy(conn->lgr->smcd))  in smcd_cdc_msg_send()
268 	diff = smc_curs_diff(conn->sndbuf_desc->len, &conn->tx_curs_fin,  in smcd_cdc_msg_send()
269 			     &conn->tx_curs_sent);  in smcd_cdc_msg_send()
272 	atomic_add(diff, &conn->sndbuf_space);  in smcd_cdc_msg_send()
275 	smc_curs_copy(&conn->tx_curs_fin, &conn->tx_curs_sent, conn);  in smcd_cdc_msg_send()
291 	struct smc_connection *conn = &smc->conn;  in smc_cdc_handle_urg_data_arrival()  local
295 	smc_curs_copy(&conn->urg_curs, &conn->local_rx_ctrl.prod, conn);  in smc_cdc_handle_urg_data_arrival()
296 	conn->urg_state = SMC_URG_VALID;  in smc_cdc_handle_urg_data_arrival()
300 	base = (char *)conn->rmb_desc->cpu_addr + conn->rx_off;  in smc_cdc_handle_urg_data_arrival()
301 	if (conn->urg_curs.count)  in smc_cdc_handle_urg_data_arrival()
302 		conn->urg_rx_byte = *(base + conn->urg_curs.count - 1);  in smc_cdc_handle_urg_data_arrival()
304 		conn->urg_rx_byte = *(base + conn->rmb_desc->len - 1);  in smc_cdc_handle_urg_data_arrival()
311 	struct smc_connection *conn = &smc->conn;  in smc_cdc_msg_validate()  local
316 	diff = conn->local_rx_ctrl.seqno - recv_seq;  in smc_cdc_msg_validate()
319 		conn->out_of_sync = 1;	/* prevent any further receives */  in smc_cdc_msg_validate()
320 		spin_lock_bh(&conn->send_lock);  in smc_cdc_msg_validate()
321 		conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1;  in smc_cdc_msg_validate()
322 		conn->lnk = link;  in smc_cdc_msg_validate()
323 		spin_unlock_bh(&conn->send_lock);  in smc_cdc_msg_validate()
325 		if (!queue_work(smc_close_wq, &conn->abort_work))  in smc_cdc_msg_validate()
334 	struct smc_connection *conn = &smc->conn;  in smc_cdc_msg_recv_action()  local
337 	smc_curs_copy(&prod_old, &conn->local_rx_ctrl.prod, conn);  in smc_cdc_msg_recv_action()
338 	smc_curs_copy(&cons_old, &conn->local_rx_ctrl.cons, conn);  in smc_cdc_msg_recv_action()
339 	smc_cdc_msg_to_host(&conn->local_rx_ctrl, cdc, conn);  in smc_cdc_msg_recv_action()
341 	diff_cons = smc_curs_diff(conn->peer_rmbe_size, &cons_old,  in smc_cdc_msg_recv_action()
342 				  &conn->local_rx_ctrl.cons);  in smc_cdc_msg_recv_action()
348 		atomic_add(diff_cons, &conn->peer_rmbe_space);  in smc_cdc_msg_recv_action()
356 		if (conn->lgr->is_smcd &&  in smc_cdc_msg_recv_action()
357 		    smc_ism_support_dmb_nocopy(conn->lgr->smcd)) {  in smc_cdc_msg_recv_action()
361 			diff_tx = smc_curs_diff(conn->sndbuf_desc->len,  in smc_cdc_msg_recv_action()
362 						&conn->tx_curs_fin,  in smc_cdc_msg_recv_action()
363 						&conn->local_rx_ctrl.cons);  in smc_cdc_msg_recv_action()
366 			atomic_add(diff_tx, &conn->sndbuf_space);  in smc_cdc_msg_recv_action()
369 			smc_curs_copy(&conn->tx_curs_fin,  in smc_cdc_msg_recv_action()
370 				      &conn->local_rx_ctrl.cons, conn);  in smc_cdc_msg_recv_action()
376 	diff_prod = smc_curs_diff(conn->rmb_desc->len, &prod_old,  in smc_cdc_msg_recv_action()
377 				  &conn->local_rx_ctrl.prod);  in smc_cdc_msg_recv_action()
379 		if (conn->local_rx_ctrl.prod_flags.urg_data_present)  in smc_cdc_msg_recv_action()
383 		atomic_add(diff_prod, &conn->bytes_to_rcv);  in smc_cdc_msg_recv_action()
388 		if (conn->local_rx_ctrl.prod_flags.write_blocked)  in smc_cdc_msg_recv_action()
390 		if (conn->local_rx_ctrl.prod_flags.urg_data_pending)  in smc_cdc_msg_recv_action()
391 			conn->urg_state = SMC_URG_NOTYET;  in smc_cdc_msg_recv_action()
395 	if ((diff_cons && smc_tx_prepared_sends(conn)) ||  in smc_cdc_msg_recv_action()
396 	    conn->local_rx_ctrl.prod_flags.cons_curs_upd_req ||  in smc_cdc_msg_recv_action()
397 	    conn->local_rx_ctrl.prod_flags.urg_data_pending) {  in smc_cdc_msg_recv_action()
399 			smc_tx_pending(conn);  in smc_cdc_msg_recv_action()
401 			conn->tx_in_release_sock = true;  in smc_cdc_msg_recv_action()
404 	if (diff_cons && conn->urg_tx_pend &&  in smc_cdc_msg_recv_action()
405 	    atomic_read(&conn->peer_rmbe_space) == conn->peer_rmbe_size) {  in smc_cdc_msg_recv_action()
407 		conn->urg_tx_pend = false;  in smc_cdc_msg_recv_action()
411 	if (conn->local_rx_ctrl.conn_state_flags.peer_conn_abort) {  in smc_cdc_msg_recv_action()
413 		conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1;  in smc_cdc_msg_recv_action()
415 	if (smc_cdc_rxed_any_close_or_senddone(conn)) {  in smc_cdc_msg_recv_action()
421 		if (!queue_work(smc_close_wq, &conn->close_work))  in smc_cdc_msg_recv_action()
444 	struct smc_connection *conn = from_tasklet(conn, t, rx_tsklet);  in smcd_cdc_rx_tsklet()  local
449 	if (!conn || conn->killed)  in smcd_cdc_rx_tsklet()
452 	data_cdc = (struct smcd_cdc_msg *)conn->rmb_desc->cpu_addr;  in smcd_cdc_rx_tsklet()
453 	smcd_curs_copy(&cdc.prod, &data_cdc->prod, conn);  in smcd_cdc_rx_tsklet()
454 	smcd_curs_copy(&cdc.cons, &data_cdc->cons, conn);  in smcd_cdc_rx_tsklet()
455 	smc = container_of(conn, struct smc_sock, conn);  in smcd_cdc_rx_tsklet()
462 void smcd_cdc_rx_init(struct smc_connection *conn)  in smcd_cdc_rx_init()  argument
464 	tasklet_setup(&conn->rx_tsklet, smcd_cdc_rx_tsklet);  in smcd_cdc_rx_init()
473 	struct smc_connection *conn;  in smc_cdc_rx_handler()  local
485 	conn = smc_lgr_find_conn(ntohl(cdc->token), lgr);  in smc_cdc_rx_handler()
487 	if (!conn || conn->out_of_sync)  in smc_cdc_rx_handler()
489 	smc = container_of(conn, struct smc_sock, conn);  in smc_cdc_rx_handler()
496 			   conn->local_rx_ctrl.seqno))  in smc_cdc_rx_handler()