1915 static void ri_pool_push(struct uprobe_task *utask, struct return_instance *ri)  in ri_pool_push()  argument
1917 	ri->cons_cnt = 0;  in ri_pool_push()
1918 	ri->next = utask->ri_pool;  in ri_pool_push()
1919 	utask->ri_pool = ri;  in ri_pool_push()
1924 	struct return_instance *ri = utask->ri_pool;  in ri_pool_pop()  local
1926 	if (likely(ri))  in ri_pool_pop()
1927 		utask->ri_pool = ri->next;  in ri_pool_pop()
1929 	return ri;  in ri_pool_pop()
1932 static void ri_free(struct return_instance *ri)  in ri_free()  argument
1934 	kfree(ri->extra_consumers);  in ri_free()
1935 	kfree_rcu(ri, rcu);  in ri_free()
1939 			      struct return_instance *ri, bool cleanup_hprobe)  in free_ret_instance()  argument
1946 		(void)hprobe_consume(&ri->hprobe, &hstate);  in free_ret_instance()
1947 		hprobe_finalize(&ri->hprobe, hstate);  in free_ret_instance()
1954 	 * processing loop isn't active, we can return ri into the pool of  in free_ret_instance()
1957 	 * just perform RCU-delated freeing of ri.  in free_ret_instance()
1963 		/* immediate reuse of ri without RCU GP is OK */  in free_ret_instance()
1964 		ri_pool_push(utask, ri);  in free_ret_instance()
1967 		ri_free(ri);  in free_ret_instance()
1978 	struct return_instance *ri, *ri_next;  in uprobe_free_utask()  local
1988 	ri = utask->return_instances;  in uprobe_free_utask()
1989 	while (ri) {  in uprobe_free_utask()
1990 		ri_next = ri->next;  in uprobe_free_utask()
1991 		free_ret_instance(utask, ri, true /* cleanup_hprobe */);  in uprobe_free_utask()
1992 		ri = ri_next;  in uprobe_free_utask()
1996 	ri = utask->ri_pool;  in uprobe_free_utask()
1997 	while (ri) {  in uprobe_free_utask()
1998 		ri_next = ri->next;  in uprobe_free_utask()
1999 		ri_free(ri);  in uprobe_free_utask()
2000 		ri = ri_next;  in uprobe_free_utask()
2014 	struct return_instance *ri;  in ri_timer()  local
2031 	for_each_ret_instance_rcu(ri, utask->return_instances)  in ri_timer()
2032 		hprobe_expire(&ri->hprobe, false);  in ri_timer()
2068 	struct return_instance *ri;  in alloc_return_instance()  local
2070 	ri = ri_pool_pop(utask);  in alloc_return_instance()
2071 	if (ri)  in alloc_return_instance()
2072 		return ri;  in alloc_return_instance()
2074 	ri = kzalloc(sizeof(*ri), GFP_KERNEL);  in alloc_return_instance()
2075 	if (!ri)  in alloc_return_instance()
2078 	return ri;  in alloc_return_instance()
2083 	struct return_instance *ri;  in dup_return_instance()  local
2085 	ri = kmemdup(old, sizeof(*ri), GFP_KERNEL);  in dup_return_instance()
2086 	if (!ri)  in dup_return_instance()
2090 		ri->extra_consumers = kmemdup(old->extra_consumers,  in dup_return_instance()
2091 					      sizeof(ri->extra_consumers[0]) * (old->cons_cnt - 1),  in dup_return_instance()
2093 		if (!ri->extra_consumers) {  in dup_return_instance()
2094 			kfree(ri);  in dup_return_instance()
2099 	return ri;  in dup_return_instance()
2208 	struct return_instance *ri = utask->return_instances, *ri_next;  in cleanup_return_instances()  local
2211 	while (ri && !arch_uretprobe_is_alive(ri, ctx, regs)) {  in cleanup_return_instances()
2212 		ri_next = ri->next;  in cleanup_return_instances()
2216 		free_ret_instance(utask, ri, true /* cleanup_hprobe */);  in cleanup_return_instances()
2217 		ri = ri_next;  in cleanup_return_instances()
2222 			      struct return_instance *ri)  in prepare_uretprobe()  argument
2268 	ri->func = instruction_pointer(regs);  in prepare_uretprobe()
2269 	ri->stack = user_stack_pointer(regs);  in prepare_uretprobe()
2270 	ri->orig_ret_vaddr = orig_ret_vaddr;  in prepare_uretprobe()
2271 	ri->chained = chained;  in prepare_uretprobe()
2275 	hprobe_init_leased(&ri->hprobe, uprobe, srcu_idx);  in prepare_uretprobe()
2276 	ri->next = utask->return_instances;  in prepare_uretprobe()
2277 	rcu_assign_pointer(utask->return_instances, ri);  in prepare_uretprobe()
2283 	ri_free(ri);  in prepare_uretprobe()
2472 static struct return_instance *push_consumer(struct return_instance *ri, __u64 id, __u64 cookie)  in push_consumer()  argument
2476 	if (unlikely(ri == ZERO_SIZE_PTR))  in push_consumer()
2477 		return ri;  in push_consumer()
2479 	if (unlikely(ri->cons_cnt > 0)) {  in push_consumer()
2480 		ric = krealloc(ri->extra_consumers, sizeof(*ric) * ri->cons_cnt, GFP_KERNEL);  in push_consumer()
2482 			ri_free(ri);  in push_consumer()
2485 		ri->extra_consumers = ric;  in push_consumer()
2488 	ric = likely(ri->cons_cnt == 0) ? &ri->consumer : &ri->extra_consumers[ri->cons_cnt - 1];  in push_consumer()
2492 	ri->cons_cnt++;  in push_consumer()
2493 	return ri;  in push_consumer()
2497 return_consumer_find(struct return_instance *ri, int *iter, int id)  in return_consumer_find()  argument
2502 	for (idx = *iter; idx < ri->cons_cnt; idx++)  in return_consumer_find()
2504 		ric = likely(idx == 0) ? &ri->consumer : &ri->extra_consumers[idx - 1];  in return_consumer_find()
2523 	struct return_instance *ri = NULL;  in handler_chain()  local
2545 		if (!ri)  in handler_chain()
2546 			ri = alloc_return_instance(utask);  in handler_chain()
2549 			ri = push_consumer(ri, uc->id, cookie);  in handler_chain()
2553 	if (!ZERO_OR_NULL_PTR(ri))  in handler_chain()
2554 		prepare_uretprobe(uprobe, regs, ri);  in handler_chain()
2570 handle_uretprobe_chain(struct return_instance *ri, struct uprobe *uprobe, struct pt_regs *regs)  in handle_uretprobe_chain()  argument
2585 			ric = return_consumer_find(ri, &ric_idx, uc->id);  in handle_uretprobe_chain()
2587 				uc->ret_handler(uc, ri->func, regs, ric ? &ric->cookie : NULL);  in handle_uretprobe_chain()
2593 static struct return_instance *find_next_ret_chain(struct return_instance *ri)  in find_next_ret_chain()  argument
2598 		chained = ri->chained;  in find_next_ret_chain()
2599 		ri = ri->next;	/* can't be NULL if chained */  in find_next_ret_chain()
2602 	return ri;  in find_next_ret_chain()
2608 	struct return_instance *ri, *ri_next, *next_chain;  in uprobe_handle_trampoline()  local
2617 	ri = utask->return_instances;  in uprobe_handle_trampoline()
2618 	if (!ri)  in uprobe_handle_trampoline()
2625 		 * or NULL; the latter case means that nobody but ri->func  in uprobe_handle_trampoline()
2628 		next_chain = find_next_ret_chain(ri);  in uprobe_handle_trampoline()
2631 		instruction_pointer_set(regs, ri->orig_ret_vaddr);  in uprobe_handle_trampoline()
2641 			ri_next = ri->next;  in uprobe_handle_trampoline()
2645 			uprobe = hprobe_consume(&ri->hprobe, &hstate);  in uprobe_handle_trampoline()
2647 				handle_uretprobe_chain(ri, uprobe, regs);  in uprobe_handle_trampoline()
2648 			hprobe_finalize(&ri->hprobe, hstate);  in uprobe_handle_trampoline()
2651 			free_ret_instance(utask, ri, false /* !cleanup_hprobe */);  in uprobe_handle_trampoline()
2652 			ri = ri_next;  in uprobe_handle_trampoline()
2653 		} while (ri != next_chain);  in uprobe_handle_trampoline()