22 static inline void __btree_path_get(struct btree_trans *trans, struct btree_path *path, bool intent)  in __btree_path_get()  argument
24 	unsigned idx = path - trans->paths;  in __btree_path_get()
26 	EBUG_ON(idx >= trans->nr_paths);  in __btree_path_get()
27 	EBUG_ON(!test_bit(idx, trans->paths_allocated));  in __btree_path_get()
29 		bch2_dump_trans_paths_updates(trans);  in __btree_path_get()
35 	trace_btree_path_get_ll(trans, path);  in __btree_path_get()
38 static inline bool __btree_path_put(struct btree_trans *trans, struct btree_path *path, bool intent)  in __btree_path_put()  argument
40 	EBUG_ON(path - trans->paths >= trans->nr_paths);  in __btree_path_put()
41 	EBUG_ON(!test_bit(path - trans->paths, trans->paths_allocated));  in __btree_path_put()
45 	trace_btree_path_put_ll(trans, path);  in __btree_path_put()
78 static inline void btree_trans_sort_paths(struct btree_trans *trans)  in btree_trans_sort_paths()  argument
81 	    trans->paths_sorted)  in btree_trans_sort_paths()
83 	__bch2_btree_trans_sort_paths(trans);  in btree_trans_sort_paths()
103 __trans_next_path(struct btree_trans *trans, unsigned *idx)  in __trans_next_path()  argument
105 	unsigned long *w = trans->paths_allocated + *idx / BITS_PER_LONG;  in __trans_next_path()
111 	while (*idx < trans->nr_paths) {  in __trans_next_path()
115 			return trans->paths + *idx;  in __trans_next_path()
138 static inline struct btree_path *next_btree_path(struct btree_trans *trans, struct btree_path *path)  in next_btree_path()  argument
142 	EBUG_ON(idx > trans->nr_sorted);  in next_btree_path()
144 	return idx < trans->nr_sorted  in next_btree_path()
145 		? trans->paths + trans->sorted[idx]  in next_btree_path()
149 static inline struct btree_path *prev_btree_path(struct btree_trans *trans, struct btree_path *path)  in prev_btree_path()  argument
151 	unsigned idx = path ? path->sorted_idx : trans->nr_sorted;  in prev_btree_path()
154 		? trans->paths + trans->sorted[idx - 1]  in prev_btree_path()
160 	     (_iter.path_idx = trans->sorted[_iter.sorted_idx],		\
171 	     (_iter.path_idx = trans->sorted[_iter.sorted_idx],		\
177 	for (_i = trans->nr_sorted - 1;					\
178 	     ((_path) = (_trans)->paths + trans->sorted[_i]), (_i) >= 0;\
189 __trans_next_path_with_node(struct btree_trans *trans, struct btree *b,  in __trans_next_path_with_node()  argument
194 	while ((path = __trans_next_path(trans, idx)) &&  in __trans_next_path_with_node()
210 bch2_btree_path_make_mut(struct btree_trans *trans,  in bch2_btree_path_make_mut()  argument
214 	if (trans->paths[path].ref > 1 ||  in bch2_btree_path_make_mut()
215 	    trans->paths[path].preserve)  in bch2_btree_path_make_mut()
216 		path = __bch2_btree_path_make_mut(trans, path, intent, ip);  in bch2_btree_path_make_mut()
217 	trans->paths[path].should_be_locked = false;  in bch2_btree_path_make_mut()
226 bch2_btree_path_set_pos(struct btree_trans *trans,  in bch2_btree_path_set_pos()  argument
230 	return !bpos_eq(new_pos, trans->paths[path].pos)  in bch2_btree_path_set_pos()
231 		? __bch2_btree_path_set_pos(trans, path, new_pos, intent, ip)  in bch2_btree_path_set_pos()
241 static inline int __must_check bch2_btree_path_traverse(struct btree_trans *trans,  in bch2_btree_path_traverse()  argument
244 	bch2_trans_verify_not_unlocked_or_in_restart(trans);  in bch2_btree_path_traverse()
246 	if (trans->paths[path].uptodate < BTREE_ITER_NEED_RELOCK)  in bch2_btree_path_traverse()
249 	return bch2_btree_path_traverse_one(trans, path, flags, _RET_IP_);  in bch2_btree_path_traverse()
282 static inline int bch2_trans_mutex_lock(struct btree_trans *trans, struct mutex *lock)  in bch2_trans_mutex_lock()  argument
286 		: __bch2_trans_mutex_lock(trans, lock);  in bch2_trans_mutex_lock()
293 static inline void bch2_trans_verify_paths(struct btree_trans *trans) {}  in bch2_trans_verify_paths()  argument
294 static inline void bch2_assert_pos_locked(struct btree_trans *trans, enum btree_id id,  in bch2_assert_pos_locked()  argument
298 void bch2_btree_path_fix_key_modified(struct btree_trans *trans,
300 void bch2_btree_node_iter_fix(struct btree_trans *trans, struct btree_path *,
313 static inline int trans_was_restarted(struct btree_trans *trans, u32 restart_count)  in trans_was_restarted()  argument
315 	return restart_count != trans->restart_count  in trans_was_restarted()
322 static inline void bch2_trans_verify_not_restarted(struct btree_trans *trans,  in bch2_trans_verify_not_restarted()  argument
325 	if (trans_was_restarted(trans, restart_count))  in bch2_trans_verify_not_restarted()
326 		bch2_trans_restart_error(trans, restart_count);  in bch2_trans_verify_not_restarted()
331 static inline void bch2_trans_verify_not_unlocked_or_in_restart(struct btree_trans *trans)  in bch2_trans_verify_not_unlocked_or_in_restart()  argument
333 	if (trans->restarted || !trans->locked)  in bch2_trans_verify_not_unlocked_or_in_restart()
334 		bch2_trans_unlocked_or_in_restart_error(trans);  in bch2_trans_verify_not_unlocked_or_in_restart()
338 static int btree_trans_restart_ip(struct btree_trans *trans, int err, unsigned long ip)  in btree_trans_restart_ip()  argument
343 	trans->restarted = err;  in btree_trans_restart_ip()
344 	trans->last_restarted_ip = ip;  in btree_trans_restart_ip()
346 	darray_exit(&trans->last_restarted_trace);  in btree_trans_restart_ip()
347 	bch2_save_backtrace(&trans->last_restarted_trace, current, 0, GFP_NOWAIT);  in btree_trans_restart_ip()
353 static int btree_trans_restart(struct btree_trans *trans, int err)  in btree_trans_restart()  argument
355 	return btree_trans_restart_ip(trans, err, _THIS_IP_);  in btree_trans_restart()
358 static inline int trans_maybe_inject_restart(struct btree_trans *trans, unsigned long ip)  in trans_maybe_inject_restart()  argument
361 	if (!(ktime_get_ns() & ~(~0ULL << min(63, (10 + trans->restart_count_this_trans))))) {  in trans_maybe_inject_restart()
362 		trace_and_count(trans->c, trans_restart_injected, trans, ip);  in trans_maybe_inject_restart()
363 		return btree_trans_restart_ip(trans,  in trans_maybe_inject_restart()
375 static inline void bch2_btree_path_downgrade(struct btree_trans *trans,  in bch2_btree_path_downgrade()  argument
381 		__bch2_btree_path_downgrade(trans, path, new_locks_want);  in bch2_btree_path_downgrade()
386 void bch2_trans_node_add(struct btree_trans *trans, struct btree_path *, struct btree *);
387 void bch2_trans_node_drop(struct btree_trans *trans, struct btree *);
432 	struct btree_trans *trans = iter->trans;  in bch2_btree_iter_set_pos()  local
435 		bch2_path_put(trans, iter->update_path,  in bch2_btree_iter_set_pos()
462 static inline unsigned bch2_btree_iter_flags(struct btree_trans *trans,  in bch2_btree_iter_flags()  argument
467 	if (level || !btree_id_cached(trans->c, btree_id)) {  in bch2_btree_iter_flags()
485 	if (trans->journal_replay_not_finished)  in bch2_btree_iter_flags()
491 static inline void bch2_trans_iter_init_common(struct btree_trans *trans,  in bch2_trans_iter_init_common()  argument
499 	iter->trans		= trans;  in bch2_trans_iter_init_common()
512 	iter->path = bch2_path_get(trans, btree_id, iter->pos,  in bch2_trans_iter_init_common()
519 static inline void bch2_trans_iter_init(struct btree_trans *trans,  in bch2_trans_iter_init()  argument
526 		bch2_trans_iter_init_common(trans, iter, btree_id, pos, 0, 0,  in bch2_trans_iter_init()
527 				bch2_btree_iter_flags(trans, btree_id, 0, flags),  in bch2_trans_iter_init()
530 		bch2_trans_iter_init_outlined(trans, iter, btree_id, pos, flags);  in bch2_trans_iter_init()
548 static inline void *bch2_trans_kmalloc(struct btree_trans *trans, size_t size)  in bch2_trans_kmalloc()  argument
552 	if (likely(trans->mem_top + size <= trans->mem_bytes)) {  in bch2_trans_kmalloc()
553 		void *p = trans->mem + trans->mem_top;  in bch2_trans_kmalloc()
555 		trans->mem_top += size;  in bch2_trans_kmalloc()
559 		return __bch2_trans_kmalloc(trans, size);  in bch2_trans_kmalloc()
563 static inline void *bch2_trans_kmalloc_nomemzero(struct btree_trans *trans, size_t size)  in bch2_trans_kmalloc_nomemzero()  argument
567 	if (likely(trans->mem_top + size <= trans->mem_bytes)) {  in bch2_trans_kmalloc_nomemzero()
568 		void *p = trans->mem + trans->mem_top;  in bch2_trans_kmalloc_nomemzero()
570 		trans->mem_top += size;  in bch2_trans_kmalloc_nomemzero()
573 		return __bch2_trans_kmalloc(trans, size);  in bch2_trans_kmalloc_nomemzero()
577 static inline struct bkey_s_c __bch2_bkey_get_iter(struct btree_trans *trans,  in __bch2_bkey_get_iter()  argument
584 	bch2_trans_iter_init(trans, iter, btree_id, pos, flags);  in __bch2_bkey_get_iter()
590 		bch2_trans_iter_exit(trans, iter);  in __bch2_bkey_get_iter()
594 static inline struct bkey_s_c bch2_bkey_get_iter(struct btree_trans *trans,  in bch2_bkey_get_iter()  argument
599 	return __bch2_bkey_get_iter(trans, iter, btree_id, pos, flags, 0);  in bch2_bkey_get_iter()
620 static inline int __bch2_bkey_get_val_typed(struct btree_trans *trans,  in __bch2_bkey_get_val_typed()  argument
626 	struct bkey_s_c k = __bch2_bkey_get_iter(trans, &iter, btree_id, pos, flags, type);  in __bch2_bkey_get_val_typed()
630 		bch2_trans_iter_exit(trans, &iter);  in __bch2_bkey_get_val_typed()
703 static inline int btree_trans_too_many_iters(struct btree_trans *trans)  in btree_trans_too_many_iters()  argument
705 	if (bitmap_weight(trans->paths_allocated, trans->nr_paths) > BTREE_ITER_NORMAL_LIMIT - 8)  in btree_trans_too_many_iters()
706 		return __bch2_btree_trans_too_many_iters(trans);  in btree_trans_too_many_iters()
784 	bch2_trans_begin(trans);					\
917 	struct btree_trans *trans = bch2_trans_get(_c);			\
919 	bch2_trans_put(trans);						\
923 #define bch2_trans_do(_c, _do)	bch2_trans_run(_c, lockrestart_do(trans, _do))