1 // SPDX-License-Identifier: GPL-2.0
32 	return seq > j->seq_ondisk;  in journal_seq_unwritten()
42 	return atomic64_read(&j->seq) - j->seq_ondisk;  in nr_unwritten_journal_entries()
47 	return __journal_entry_is_open(j->reservations);  in journal_entry_is_open()
52 	union journal_res_state s = READ_ONCE(j->reservations);  in bch2_journal_buf_to_text()
54 	struct journal_buf *buf = j->buf + i;  in bch2_journal_buf_to_text()
62 	prt_human_readable_u64(out, vstruct_bytes(buf->data));  in bch2_journal_buf_to_text()
66 	prt_printf(out, "%li jiffies\n", buf->expires - jiffies);  in bch2_journal_buf_to_text()
69 	if (buf->noflush)  in bch2_journal_buf_to_text()
71 	if (buf->must_flush)  in bch2_journal_buf_to_text()
73 	if (buf->separate_flush)  in bch2_journal_buf_to_text()
75 	if (buf->need_flush_to_write_buffer)  in bch2_journal_buf_to_text()
77 	if (buf->write_started)  in bch2_journal_buf_to_text()
79 	if (buf->write_allocated)  in bch2_journal_buf_to_text()
81 	if (buf->write_done)  in bch2_journal_buf_to_text()
90 	if (!out->nr_tabstops)  in bch2_journal_bufs_to_text()
108 		buf = j->buf + (seq & JOURNAL_BUF_MASK);  in journal_seq_to_buf()
109 		EBUG_ON(le64_to_cpu(buf->data->seq) != seq);  in journal_seq_to_buf()
116 	for (unsigned i = 0; i < ARRAY_SIZE(p->unflushed); i++)  in journal_pin_list_init()
117 		INIT_LIST_HEAD(&p->unflushed[i]);  in journal_pin_list_init()
118 	for (unsigned i = 0; i < ARRAY_SIZE(p->flushed); i++)  in journal_pin_list_init()
119 		INIT_LIST_HEAD(&p->flushed[i]);  in journal_pin_list_init()
120 	atomic_set(&p->count, count);  in journal_pin_list_init()
121 	p->devs.nr = 0;  in journal_pin_list_init()
148 	spin_lock(&j->lock);  in journal_error_check_stuck()
150 	if (j->can_discard) {  in journal_error_check_stuck()
151 		spin_unlock(&j->lock);  in journal_error_check_stuck()
158 	 * The journal shutdown path will set ->err_seq, but do it here first to  in journal_error_check_stuck()
162 	if (j->err_seq) {  in journal_error_check_stuck()
163 		spin_unlock(&j->lock);  in journal_error_check_stuck()
166 	j->err_seq = journal_cur_seq(j);  in journal_error_check_stuck()
167 	spin_unlock(&j->lock);  in journal_error_check_stuck()
169 	bch_err(c, "Journal stuck! Hava a pre-reservation but journal full (error %s)",  in journal_error_check_stuck()
191 		struct journal_buf *w = j->buf + idx;  in bch2_journal_do_writes()
193 		if (w->write_started && !w->write_allocated)  in bch2_journal_do_writes()
195 		if (w->write_started)  in bch2_journal_do_writes()
198 		if (!journal_state_count(j->reservations, idx)) {  in bch2_journal_do_writes()
199 			w->write_started = true;  in bch2_journal_do_writes()
200 			closure_call(&w->io, bch2_journal_write, j->wq, NULL);  in bch2_journal_do_writes()
209  * dropped. Drop the pin list reference acquired at journal entry open and write
214 	lockdep_assert_held(&j->lock);  in bch2_journal_buf_put_final()
224 	wake_up(&j->wait);  in bch2_journal_buf_put_final()
231  * and can be opened again - this also initializes the next journal_buf:
243 	lockdep_assert_held(&j->lock);  in __journal_entry_close()
245 	old.v = atomic64_read(&j->reservations.counter);  in __journal_entry_close()
253 	} while (!atomic64_try_cmpxchg(&j->reservations.counter,  in __journal_entry_close()
260 		old.cur_entry_offset = j->cur_entry_offset_if_blocked;  in __journal_entry_close()
263 	buf->data->u64s		= cpu_to_le32(old.cur_entry_offset);  in __journal_entry_close()
270 		prt_human_readable_u64(&pbuf, vstruct_bytes(buf->data));  in __journal_entry_close()
277 	sectors = vstruct_blocks_plus(buf->data, c->block_bits,  in __journal_entry_close()
278 				      buf->u64s_reserved) << c->block_bits;  in __journal_entry_close()
279 	BUG_ON(sectors > buf->sectors);  in __journal_entry_close()
280 	buf->sectors = sectors;  in __journal_entry_close()
285 	 * A threads may replace an old pin with a new pin on their current  in __journal_entry_close()
286 	 * journal reservation - the expectation being that the journal will  in __journal_entry_close()
287 	 * contain either what the old pin protected or what the new pin  in __journal_entry_close()
290 	 * After the old pin is dropped journal_last_seq() won't include the old  in __journal_entry_close()
291 	 * pin, so we can only write the updated last_seq on the entry that  in __journal_entry_close()
292 	 * contains whatever the new pin protects.  in __journal_entry_close()
301 	buf->last_seq		= journal_last_seq(j);  in __journal_entry_close()
302 	buf->data->last_seq	= cpu_to_le64(buf->last_seq);  in __journal_entry_close()
303 	BUG_ON(buf->last_seq > le64_to_cpu(buf->data->seq));  in __journal_entry_close()
305 	cancel_delayed_work(&j->write_work);  in __journal_entry_close()
309 	__bch2_journal_buf_put(j, old.idx, le64_to_cpu(buf->data->seq));  in __journal_entry_close()
314 	spin_lock(&j->lock);  in bch2_journal_halt()
316 	if (!j->err_seq)  in bch2_journal_halt()
317 		j->err_seq = journal_cur_seq(j);  in bch2_journal_halt()
319 	spin_unlock(&j->lock);  in bch2_journal_halt()
324 	lockdep_assert_held(&j->lock);  in bch2_journal_halt_locked()
327 	if (!j->err_seq)  in bch2_journal_halt_locked()
328 		j->err_seq = journal_cur_seq(j);  in bch2_journal_halt_locked()
343 		if (!buf->flush_time) {  in journal_entry_want_write()
344 			buf->flush_time	= local_clock() ?: 1;  in journal_entry_want_write()
345 			buf->expires = jiffies;  in journal_entry_want_write()
356 	spin_lock(&j->lock);  in bch2_journal_entry_close()
358 	spin_unlock(&j->lock);  in bch2_journal_entry_close()
364  * should _only_ called from journal_res_get() - when we actually want a
365  * journal reservation - journal entry is open means journal is dirty:
370 	struct journal_buf *buf = j->buf +  in journal_entry_open()
375 	lockdep_assert_held(&j->lock);  in journal_entry_open()
377 	BUG_ON(BCH_SB_CLEAN(c->disk_sb.sb));  in journal_entry_open()
379 	if (j->blocked)  in journal_entry_open()
382 	if (j->cur_entry_error)  in journal_entry_open()
383 		return j->cur_entry_error;  in journal_entry_open()
386 		return JOURNAL_ERR_insufficient_devices; /* -EROFS */  in journal_entry_open()
388 	if (!fifo_free(&j->pin))  in journal_entry_open()
391 	if (nr_unwritten_journal_entries(j) == ARRAY_SIZE(j->buf))  in journal_entry_open()
397 			bch_err(c, "fatal error - emergency read only");  in journal_entry_open()
398 		return JOURNAL_ERR_insufficient_devices; /* -EROFS */  in journal_entry_open()
401 	BUG_ON(!j->cur_entry_sectors);  in journal_entry_open()
403 	buf->expires		=  in journal_entry_open()
404 		(journal_cur_seq(j) == j->flushed_seq_ondisk  in journal_entry_open()
406 		 : j->last_flush_write) +  in journal_entry_open()
407 		msecs_to_jiffies(c->opts.journal_flush_delay);  in journal_entry_open()
409 	buf->u64s_reserved	= j->entry_u64s_reserved;  in journal_entry_open()
410 	buf->disk_sectors	= j->cur_entry_sectors;  in journal_entry_open()
411 	buf->sectors		= min(buf->disk_sectors, buf->buf_size >> 9);  in journal_entry_open()
413 	u64s = (int) (buf->sectors << 9) / sizeof(u64) -  in journal_entry_open()
415 	u64s = clamp_t(int, u64s, 0, JOURNAL_ENTRY_CLOSED_VAL - 1);  in journal_entry_open()
417 	if (u64s <= (ssize_t) j->early_journal_entries.nr)  in journal_entry_open()
420 	if (fifo_empty(&j->pin) && j->reclaim_thread)  in journal_entry_open()
421 		wake_up_process(j->reclaim_thread);  in journal_entry_open()
424 	 * The fifo_push() needs to happen at the same time as j->seq is  in journal_entry_open()
427 	atomic64_inc(&j->seq);  in journal_entry_open()
428 	journal_pin_list_init(fifo_push_ref(&j->pin), 1);  in journal_entry_open()
430 	BUG_ON(j->pin.back - 1 != atomic64_read(&j->seq));  in journal_entry_open()
432 	BUG_ON(j->buf + (journal_cur_seq(j) & JOURNAL_BUF_MASK) != buf);  in journal_entry_open()
434 	bkey_extent_init(&buf->key);  in journal_entry_open()
435 	buf->noflush		= false;  in journal_entry_open()
436 	buf->must_flush		= false;  in journal_entry_open()
437 	buf->separate_flush	= false;  in journal_entry_open()
438 	buf->flush_time		= 0;  in journal_entry_open()
439 	buf->need_flush_to_write_buffer = true;  in journal_entry_open()
440 	buf->write_started	= false;  in journal_entry_open()
441 	buf->write_allocated	= false;  in journal_entry_open()
442 	buf->write_done		= false;  in journal_entry_open()
444 	memset(buf->data, 0, sizeof(*buf->data));  in journal_entry_open()
445 	buf->data->seq	= cpu_to_le64(journal_cur_seq(j));  in journal_entry_open()
446 	buf->data->u64s	= 0;  in journal_entry_open()
448 	if (j->early_journal_entries.nr) {  in journal_entry_open()
449 		memcpy(buf->data->_data, j->early_journal_entries.data,  in journal_entry_open()
450 		       j->early_journal_entries.nr * sizeof(u64));  in journal_entry_open()
451 		le32_add_cpu(&buf->data->u64s, j->early_journal_entries.nr);  in journal_entry_open()
457 	j->cur_entry_u64s = u64s;  in journal_entry_open()
459 	old.v = atomic64_read(&j->reservations.counter);  in journal_entry_open()
472 		new.cur_entry_offset = le32_to_cpu(buf->data->u64s);  in journal_entry_open()
473 	} while (!atomic64_try_cmpxchg(&j->reservations.counter,  in journal_entry_open()
477 		mod_delayed_work(j->wq,  in journal_entry_open()
478 				 &j->write_work,  in journal_entry_open()
479 				 msecs_to_jiffies(c->opts.journal_flush_delay));  in journal_entry_open()
482 	if (j->early_journal_entries.nr)  in journal_entry_open()
483 		darray_exit(&j->early_journal_entries);  in journal_entry_open()
489 	bool ret = atomic64_read(&j->seq) == j->seq_ondisk;  in journal_quiesced()
498 	wait_event(j->wait, journal_quiesced(j));  in journal_quiesce()
505 	spin_lock(&j->lock);  in journal_write_work()
506 	if (__journal_entry_is_open(j->reservations)) {  in journal_write_work()
507 		long delta = journal_cur_buf(j)->expires - jiffies;  in journal_write_work()
510 			mod_delayed_work(j->wq, &j->write_work, delta);  in journal_write_work()
514 	spin_unlock(&j->lock);  in journal_write_work()
529 		return -BCH_ERR_erofs_journal_err;  in __journal_res_get()
531 	if (j->blocked)  in __journal_res_get()
532 		return -BCH_ERR_journal_res_get_blocked;  in __journal_res_get()
534 	if ((flags & BCH_WATERMARK_MASK) < j->watermark) {  in __journal_res_get()
536 		can_discard = j->can_discard;  in __journal_res_get()
540 	if (nr_unwritten_journal_entries(j) == ARRAY_SIZE(j->buf) && !journal_entry_is_open(j)) {  in __journal_res_get()
545 	spin_lock(&j->lock);  in __journal_res_get()
564 	    buf->buf_size >> 9 < buf->disk_sectors &&  in __journal_res_get()
565 	    buf->buf_size < JOURNAL_ENTRY_SIZE_MAX)  in __journal_res_get()
566 		j->buf_size_want = max(j->buf_size_want, buf->buf_size << 1);  in __journal_res_get()
571 	can_discard = j->can_discard;  in __journal_res_get()
572 	spin_unlock(&j->lock);  in __journal_res_get()
580 		ret = -BCH_ERR_journal_res_get_blocked;  in __journal_res_get()
583 	    track_event_change(&c->times[BCH_TIME_blocked_journal_max_in_flight], true)) {  in __journal_res_get()
594 	 * Journal is full - can't rely on reclaim from work item due to  in __journal_res_get()
605 		if (mutex_trylock(&j->reclaim_lock)) {  in __journal_res_get()
607 			mutex_unlock(&j->reclaim_lock);  in __journal_res_get()
612 		? -BCH_ERR_erofs_journal_err  in __journal_res_get()
613 		: -BCH_ERR_journal_res_get_blocked;  in __journal_res_get()
621 		nsecs = max(nsecs, ca->io_latency[WRITE].stats.max_duration);  in max_dev_latency()
642 	if (closure_wait_event_timeout(&j->async_wait,  in bch2_journal_res_get_slowpath()
643 		   (ret = __journal_res_get(j, res, flags)) != -BCH_ERR_journal_res_get_blocked ||  in bch2_journal_res_get_slowpath()
654 	remaining_wait = max(0, remaining_wait - HZ);  in bch2_journal_res_get_slowpath()
656 	if (closure_wait_event_timeout(&j->async_wait,  in bch2_journal_res_get_slowpath()
657 		   (ret = __journal_res_get(j, res, flags)) != -BCH_ERR_journal_res_get_blocked ||  in bch2_journal_res_get_slowpath()
668 	closure_wait_event(&j->async_wait,  in bch2_journal_res_get_slowpath()
669 		   (ret = __journal_res_get(j, res, flags)) != -BCH_ERR_journal_res_get_blocked ||  in bch2_journal_res_get_slowpath()
681 	int d = new_u64s - res->u64s;  in bch2_journal_entry_res_resize()
683 	spin_lock(&j->lock);  in bch2_journal_entry_res_resize()
685 	j->entry_u64s_reserved += d;  in bch2_journal_entry_res_resize()
689 	j->cur_entry_u64s = max_t(int, 0, j->cur_entry_u64s - d);  in bch2_journal_entry_res_resize()
691 	state = READ_ONCE(j->reservations);  in bch2_journal_entry_res_resize()
694 	    state.cur_entry_offset > j->cur_entry_u64s) {  in bch2_journal_entry_res_resize()
695 		j->cur_entry_u64s += d;  in bch2_journal_entry_res_resize()
701 		journal_cur_buf(j)->u64s_reserved += d;  in bch2_journal_entry_res_resize()
704 	spin_unlock(&j->lock);  in bch2_journal_entry_res_resize()
705 	res->u64s += d;  in bch2_journal_entry_res_resize()
711  * bch2_journal_flush_seq_async - wait for a journal entry to be written
714  * @parent:	closure object to wait with
716  *		-BCH_ERR_journal_flush_err if @seq will never be flushed
727 	if (seq <= j->flushed_seq_ondisk)  in bch2_journal_flush_seq_async()
730 	spin_lock(&j->lock);  in bch2_journal_flush_seq_async()
738 	if (j->err_seq && seq >= j->err_seq) {  in bch2_journal_flush_seq_async()
739 		ret = -BCH_ERR_journal_flush_err;  in bch2_journal_flush_seq_async()
743 	if (seq <= j->flushed_seq_ondisk) {  in bch2_journal_flush_seq_async()
748 	/* if seq was written, but not flushed - flush a newer one instead */  in bch2_journal_flush_seq_async()
758 		spin_unlock(&j->lock);  in bch2_journal_flush_seq_async()
761 		 * We're called from bch2_journal_flush_seq() -> wait_event();  in bch2_journal_flush_seq_async()
772 		buf->must_flush = true;  in bch2_journal_flush_seq_async()
774 		if (!buf->flush_time) {  in bch2_journal_flush_seq_async()
775 			buf->flush_time	= local_clock() ?: 1;  in bch2_journal_flush_seq_async()
776 			buf->expires = jiffies;  in bch2_journal_flush_seq_async()
779 		if (parent && !closure_wait(&buf->wait, parent))  in bch2_journal_flush_seq_async()
784 		spin_lock(&j->lock);  in bch2_journal_flush_seq_async()
793 	if (buf->noflush) {  in bch2_journal_flush_seq_async()
798 	buf->must_flush = true;  in bch2_journal_flush_seq_async()
799 	j->flushing_seq = max(j->flushing_seq, seq);  in bch2_journal_flush_seq_async()
801 	if (parent && !closure_wait(&buf->wait, parent))  in bch2_journal_flush_seq_async()
807 	spin_unlock(&j->lock);  in bch2_journal_flush_seq_async()
819 	if (seq <= j->flushed_seq_ondisk)  in bch2_journal_flush_seq()
822 	ret = wait_event_state(j->wait,  in bch2_journal_flush_seq()
827 		bch2_time_stats_update(j->flush_seq_time, start_time);  in bch2_journal_flush_seq()
833  * bch2_journal_flush_async - if there is an open journal entry, or a journal
834  * still being written, write it and wait for the write to complete
838 	bch2_journal_flush_seq_async(j, atomic64_read(&j->seq), parent);  in bch2_journal_flush_async()
843 	return bch2_journal_flush_seq(j, atomic64_read(&j->seq), TASK_UNINTERRUPTIBLE);  in bch2_journal_flush()
847  * bch2_journal_noflush_seq - ask the journal not to issue any flushes in the
857 	if (!(c->sb.features & (1ULL << BCH_FEATURE_journal_no_flush)))  in bch2_journal_noflush_seq()
860 	if (c->journal.flushed_seq_ondisk >= start)  in bch2_journal_noflush_seq()
863 	spin_lock(&j->lock);  in bch2_journal_noflush_seq()
864 	if (c->journal.flushed_seq_ondisk >= start)  in bch2_journal_noflush_seq()
873 		if (buf->must_flush)  in bch2_journal_noflush_seq()
876 		buf->noflush = true;  in bch2_journal_noflush_seq()
881 	spin_unlock(&j->lock);  in bch2_journal_noflush_seq()
892 	struct journal_buf *buf = j->buf + (res.seq & JOURNAL_BUF_MASK);  in __bch2_journal_meta()
893 	buf->must_flush = true;  in __bch2_journal_meta()
895 	if (!buf->flush_time) {  in __bch2_journal_meta()
896 		buf->flush_time	= local_clock() ?: 1;  in __bch2_journal_meta()
897 		buf->expires = jiffies;  in __bch2_journal_meta()
910 		return -EROFS;  in bch2_journal_meta()
921 	spin_lock(&j->lock);  in bch2_journal_unblock()
922 	if (!--j->blocked &&  in bch2_journal_unblock()
923 	    j->cur_entry_offset_if_blocked < JOURNAL_ENTRY_CLOSED_VAL &&  in bch2_journal_unblock()
924 	    j->reservations.cur_entry_offset == JOURNAL_ENTRY_BLOCKED_VAL) {  in bch2_journal_unblock()
927 		old.v = atomic64_read(&j->reservations.counter);  in bch2_journal_unblock()
930 			new.cur_entry_offset = j->cur_entry_offset_if_blocked;  in bch2_journal_unblock()
931 		} while (!atomic64_try_cmpxchg(&j->reservations.counter, &old.v, new.v));  in bch2_journal_unblock()
933 	spin_unlock(&j->lock);  in bch2_journal_unblock()
940 	if (!j->blocked++) {  in __bch2_journal_block()
943 		old.v = atomic64_read(&j->reservations.counter);  in __bch2_journal_block()
945 			j->cur_entry_offset_if_blocked = old.cur_entry_offset;  in __bch2_journal_block()
947 			if (j->cur_entry_offset_if_blocked >= JOURNAL_ENTRY_CLOSED_VAL)  in __bch2_journal_block()
952 		} while (!atomic64_try_cmpxchg(&j->reservations.counter, &old.v, new.v));  in __bch2_journal_block()
954 		journal_cur_buf(j)->data->u64s = cpu_to_le32(old.cur_entry_offset);  in __bch2_journal_block()
960 	spin_lock(&j->lock);  in bch2_journal_block()
962 	spin_unlock(&j->lock);  in bch2_journal_block()
974 	mutex_lock(&j->buf_lock);  in __bch2_next_write_buffer_flush_journal_buf()
975 	spin_lock(&j->lock);  in __bch2_next_write_buffer_flush_journal_buf()
982 		struct journal_buf *buf = j->buf + idx;  in __bch2_next_write_buffer_flush_journal_buf()
984 		if (buf->need_flush_to_write_buffer) {  in __bch2_next_write_buffer_flush_journal_buf()
986 			s.v = atomic64_read_acquire(&j->reservations.counter);  in __bch2_next_write_buffer_flush_journal_buf()
996 				? ERR_PTR(-EAGAIN)  in __bch2_next_write_buffer_flush_journal_buf()
1002 	spin_unlock(&j->lock);  in __bch2_next_write_buffer_flush_journal_buf()
1004 		mutex_unlock(&j->buf_lock);  in __bch2_next_write_buffer_flush_journal_buf()
1014 	wait_event(j->wait, (ret = __bch2_next_write_buffer_flush_journal_buf(j,  in bch2_next_write_buffer_flush_journal_buf()
1015 						max_seq, blocked)) != ERR_PTR(-EAGAIN));  in bch2_next_write_buffer_flush_journal_buf()
1027 	struct bch_fs *c = ca->fs;  in bch2_set_nr_journal_buckets_iter()
1028 	struct journal_device *ja = &ca->journal;  in bch2_set_nr_journal_buckets_iter()
1032 	unsigned i, pos, nr_got = 0, nr_want = nr - ja->nr;  in bch2_set_nr_journal_buckets_iter()
1035 	BUG_ON(nr <= ja->nr);  in bch2_set_nr_journal_buckets_iter()
1042 		ret = -BCH_ERR_ENOMEM_set_nr_journal_buckets;  in bch2_set_nr_journal_buckets_iter()
1060 						ob[nr_got]->bucket, BCH_DATA_journal,  in bch2_set_nr_journal_buckets_iter()
1061 						ca->mi.bucket_size, BTREE_TRIGGER_transactional));  in bch2_set_nr_journal_buckets_iter()
1069 		bu[nr_got] = ob[nr_got]->bucket;  in bch2_set_nr_journal_buckets_iter()
1079 		bch2_journal_flush_all_pins(&c->journal);  in bch2_set_nr_journal_buckets_iter()
1080 		bch2_journal_block(&c->journal);  in bch2_set_nr_journal_buckets_iter()
1081 		mutex_lock(&c->sb_lock);  in bch2_set_nr_journal_buckets_iter()
1084 	memcpy(new_buckets,	ja->buckets,	ja->nr * sizeof(u64));  in bch2_set_nr_journal_buckets_iter()
1085 	memcpy(new_bucket_seq,	ja->bucket_seq,	ja->nr * sizeof(u64));  in bch2_set_nr_journal_buckets_iter()
1087 	BUG_ON(ja->discard_idx > ja->nr);  in bch2_set_nr_journal_buckets_iter()
1089 	pos = ja->discard_idx ?: ja->nr;  in bch2_set_nr_journal_buckets_iter()
1093 		sizeof(new_buckets[0]) * (ja->nr - pos));  in bch2_set_nr_journal_buckets_iter()
1096 		sizeof(new_bucket_seq[0]) * (ja->nr - pos));  in bch2_set_nr_journal_buckets_iter()
1103 	nr = ja->nr + nr_got;  in bch2_set_nr_journal_buckets_iter()
1113 		spin_lock(&c->journal.lock);  in bch2_set_nr_journal_buckets_iter()
1115 	swap(new_buckets,	ja->buckets);  in bch2_set_nr_journal_buckets_iter()
1116 	swap(new_bucket_seq,	ja->bucket_seq);  in bch2_set_nr_journal_buckets_iter()
1117 	ja->nr = nr;  in bch2_set_nr_journal_buckets_iter()
1119 	if (pos <= ja->discard_idx)  in bch2_set_nr_journal_buckets_iter()
1120 		ja->discard_idx = (ja->discard_idx + nr_got) % ja->nr;  in bch2_set_nr_journal_buckets_iter()
1121 	if (pos <= ja->dirty_idx_ondisk)  in bch2_set_nr_journal_buckets_iter()
1122 		ja->dirty_idx_ondisk = (ja->dirty_idx_ondisk + nr_got) % ja->nr;  in bch2_set_nr_journal_buckets_iter()
1123 	if (pos <= ja->dirty_idx)  in bch2_set_nr_journal_buckets_iter()
1124 		ja->dirty_idx = (ja->dirty_idx + nr_got) % ja->nr;  in bch2_set_nr_journal_buckets_iter()
1125 	if (pos <= ja->cur_idx)  in bch2_set_nr_journal_buckets_iter()
1126 		ja->cur_idx = (ja->cur_idx + nr_got) % ja->nr;  in bch2_set_nr_journal_buckets_iter()
1129 		spin_unlock(&c->journal.lock);  in bch2_set_nr_journal_buckets_iter()
1132 		bch2_journal_unblock(&c->journal);  in bch2_set_nr_journal_buckets_iter()
1133 		mutex_unlock(&c->sb_lock);  in bch2_set_nr_journal_buckets_iter()
1156 	struct journal_device *ja = &ca->journal;  in bch2_set_nr_journal_buckets_loop()
1163 	if (nr < ja->nr)  in bch2_set_nr_journal_buckets_loop()
1166 	while (!ret && ja->nr < nr) {  in bch2_set_nr_journal_buckets_loop()
1172 		 * when space used goes up without a reservation - but we do need the  in bch2_set_nr_journal_buckets_loop()
1176 		 * filesystem-wide allocation will succeed, this is a device  in bch2_set_nr_journal_buckets_loop()
1177 		 * specific allocation - we can hang here:  in bch2_set_nr_journal_buckets_loop()
1181 							bucket_to_sector(ca, nr - ja->nr), 1, 0);  in bch2_set_nr_journal_buckets_loop()
1188 		if (ret == -BCH_ERR_bucket_alloc_blocked ||  in bch2_set_nr_journal_buckets_loop()
1189 		    ret == -BCH_ERR_open_buckets_empty)  in bch2_set_nr_journal_buckets_loop()
1190 			ret = 0; /* wait and retry */  in bch2_set_nr_journal_buckets_loop()
1200  * Allocate more journal space at runtime - not currently making use if it, but
1206 	down_write(&c->state_lock);  in bch2_set_nr_journal_buckets()
1208 	up_write(&c->state_lock);  in bch2_set_nr_journal_buckets()
1220 		ret = -BCH_ERR_ENOMEM_set_nr_journal_buckets;  in bch2_dev_journal_alloc()
1225 	nr = ca->mi.nbuckets >> 7;  in bch2_dev_journal_alloc()
1234 			 (1 << 24) / ca->mi.bucket_size));  in bch2_dev_journal_alloc()
1236 	ret = bch2_set_nr_journal_buckets_loop(ca->fs, ca, nr, new_fs);  in bch2_dev_journal_alloc()
1245 		if (ca->journal.nr)  in bch2_fs_journal_alloc()
1250 			percpu_ref_put(&ca->io_ref);  in bch2_fs_journal_alloc()
1265 	spin_lock(&j->lock);  in bch2_journal_writing_to_device()
1271 		if (bch2_bkey_has_device_c(bkey_i_to_s_c(&buf->key), dev_idx))  in bch2_journal_writing_to_device()
1274 	spin_unlock(&j->lock);  in bch2_journal_writing_to_device()
1281 	wait_event(j->wait, !bch2_journal_writing_to_device(j, ca->dev_idx));  in bch2_dev_journal_stop()
1286 	if (!test_bit(JOURNAL_running, &j->flags))  in bch2_fs_journal_stop()
1292 	wait_event(j->wait, bch2_journal_entry_close(j));  in bch2_fs_journal_stop()
1301 	cancel_delayed_work_sync(&j->write_work);  in bch2_fs_journal_stop()
1304 	     test_bit(JOURNAL_replay_done, &j->flags) &&  in bch2_fs_journal_stop()
1305 	     j->last_empty_seq != journal_cur_seq(j),  in bch2_fs_journal_stop()
1307 	     journal_cur_seq(j), j->last_empty_seq);  in bch2_fs_journal_stop()
1310 		clear_bit(JOURNAL_running, &j->flags);  in bch2_fs_journal_stop()
1324 		return -EINVAL;  in bch2_fs_journal_start()
1327 	genradix_for_each_reverse(&c->journal_entries, iter, _i) {  in bch2_fs_journal_start()
1333 		last_seq = le64_to_cpu(i->j.last_seq);  in bch2_fs_journal_start()
1337 	nr = cur_seq - last_seq;  in bch2_fs_journal_start()
1339 	if (nr + 1 > j->pin.size) {  in bch2_fs_journal_start()
1340 		free_fifo(&j->pin);  in bch2_fs_journal_start()
1341 		init_fifo(&j->pin, roundup_pow_of_two(nr + 1), GFP_KERNEL);  in bch2_fs_journal_start()
1342 		if (!j->pin.data) {  in bch2_fs_journal_start()
1344 			return -BCH_ERR_ENOMEM_journal_pin_fifo;  in bch2_fs_journal_start()
1348 	j->replay_journal_seq	= last_seq;  in bch2_fs_journal_start()
1349 	j->replay_journal_seq_end = cur_seq;  in bch2_fs_journal_start()
1350 	j->last_seq_ondisk	= last_seq;  in bch2_fs_journal_start()
1351 	j->flushed_seq_ondisk	= cur_seq - 1;  in bch2_fs_journal_start()
1352 	j->seq_ondisk		= cur_seq - 1;  in bch2_fs_journal_start()
1353 	j->pin.front		= last_seq;  in bch2_fs_journal_start()
1354 	j->pin.back		= cur_seq;  in bch2_fs_journal_start()
1355 	atomic64_set(&j->seq, cur_seq - 1);  in bch2_fs_journal_start()
1357 	fifo_for_each_entry_ptr(p, &j->pin, seq)  in bch2_fs_journal_start()
1360 	genradix_for_each(&c->journal_entries, iter, _i) {  in bch2_fs_journal_start()
1366 		seq = le64_to_cpu(i->j.seq);  in bch2_fs_journal_start()
1372 		if (journal_entry_empty(&i->j))  in bch2_fs_journal_start()
1373 			j->last_empty_seq = le64_to_cpu(i->j.seq);  in bch2_fs_journal_start()
1377 		p->devs.nr = 0;  in bch2_fs_journal_start()
1378 		darray_for_each(i->ptrs, ptr)  in bch2_fs_journal_start()
1379 			bch2_dev_list_add_dev(&p->devs, ptr->dev);  in bch2_fs_journal_start()
1385 		j->last_empty_seq = cur_seq - 1; /* to match j->seq */  in bch2_fs_journal_start()
1387 	spin_lock(&j->lock);  in bch2_fs_journal_start()
1389 	set_bit(JOURNAL_running, &j->flags);  in bch2_fs_journal_start()
1390 	j->last_flush_write = jiffies;  in bch2_fs_journal_start()
1392 	j->reservations.idx = j->reservations.unwritten_idx = journal_cur_seq(j);  in bch2_fs_journal_start()
1393 	j->reservations.unwritten_idx++;  in bch2_fs_journal_start()
1395 	c->last_bucket_seq_cleanup = journal_cur_seq(j);  in bch2_fs_journal_start()
1398 	spin_unlock(&j->lock);  in bch2_fs_journal_start()
1407 	struct journal_device *ja = &ca->journal;  in bch2_dev_journal_exit()
1409 	for (unsigned i = 0; i < ARRAY_SIZE(ja->bio); i++) {  in bch2_dev_journal_exit()
1410 		kfree(ja->bio[i]);  in bch2_dev_journal_exit()
1411 		ja->bio[i] = NULL;  in bch2_dev_journal_exit()
1414 	kfree(ja->buckets);  in bch2_dev_journal_exit()
1415 	kfree(ja->bucket_seq);  in bch2_dev_journal_exit()
1416 	ja->buckets	= NULL;  in bch2_dev_journal_exit()
1417 	ja->bucket_seq	= NULL;  in bch2_dev_journal_exit()
1422 	struct journal_device *ja = &ca->journal;  in bch2_dev_journal_init()
1428 	ja->nr = 0;  in bch2_dev_journal_init()
1434 			ja->nr += le64_to_cpu(journal_buckets_v2->d[i].nr);  in bch2_dev_journal_init()
1436 		ja->nr = bch2_nr_journal_buckets(journal_buckets);  in bch2_dev_journal_init()
1439 	ja->bucket_seq = kcalloc(ja->nr, sizeof(u64), GFP_KERNEL);  in bch2_dev_journal_init()
1440 	if (!ja->bucket_seq)  in bch2_dev_journal_init()
1441 		return -BCH_ERR_ENOMEM_dev_journal_init;  in bch2_dev_journal_init()
1445 	for (unsigned i = 0; i < ARRAY_SIZE(ja->bio); i++) {  in bch2_dev_journal_init()
1446 		ja->bio[i] = kmalloc(struct_size(ja->bio[i], bio.bi_inline_vecs,  in bch2_dev_journal_init()
1448 		if (!ja->bio[i])  in bch2_dev_journal_init()
1449 			return -BCH_ERR_ENOMEM_dev_journal_init;  in bch2_dev_journal_init()
1451 		ja->bio[i]->ca = ca;  in bch2_dev_journal_init()
1452 		ja->bio[i]->buf_idx = i;  in bch2_dev_journal_init()
1453 		bio_init(&ja->bio[i]->bio, NULL, ja->bio[i]->bio.bi_inline_vecs, nr_bvecs, 0);  in bch2_dev_journal_init()
1456 	ja->buckets = kcalloc(ja->nr, sizeof(u64), GFP_KERNEL);  in bch2_dev_journal_init()
1457 	if (!ja->buckets)  in bch2_dev_journal_init()
1458 		return -BCH_ERR_ENOMEM_dev_journal_init;  in bch2_dev_journal_init()
1465 			for (unsigned j = 0; j < le64_to_cpu(journal_buckets_v2->d[i].nr); j++)  in bch2_dev_journal_init()
1466 				ja->buckets[dst++] =  in bch2_dev_journal_init()
1467 					le64_to_cpu(journal_buckets_v2->d[i].start) + j;  in bch2_dev_journal_init()
1469 		for (unsigned i = 0; i < ja->nr; i++)  in bch2_dev_journal_init()
1470 			ja->buckets[i] = le64_to_cpu(journal_buckets->buckets[i]);  in bch2_dev_journal_init()
1478 	if (j->wq)  in bch2_fs_journal_exit()
1479 		destroy_workqueue(j->wq);  in bch2_fs_journal_exit()
1481 	darray_exit(&j->early_journal_entries);  in bch2_fs_journal_exit()
1483 	for (unsigned i = 0; i < ARRAY_SIZE(j->buf); i++)  in bch2_fs_journal_exit()
1484 		kvfree(j->buf[i].data);  in bch2_fs_journal_exit()
1485 	free_fifo(&j->pin);  in bch2_fs_journal_exit()
1492 	mutex_init(&j->buf_lock);  in bch2_fs_journal_init()
1493 	spin_lock_init(&j->lock);  in bch2_fs_journal_init()
1494 	spin_lock_init(&j->err_lock);  in bch2_fs_journal_init()
1495 	init_waitqueue_head(&j->wait);  in bch2_fs_journal_init()
1496 	INIT_DELAYED_WORK(&j->write_work, journal_write_work);  in bch2_fs_journal_init()
1497 	init_waitqueue_head(&j->reclaim_wait);  in bch2_fs_journal_init()
1498 	init_waitqueue_head(&j->pin_flush_wait);  in bch2_fs_journal_init()
1499 	mutex_init(&j->reclaim_lock);  in bch2_fs_journal_init()
1500 	mutex_init(&j->discard_lock);  in bch2_fs_journal_init()
1502 	lockdep_init_map(&j->res_map, "journal res", &res_key, 0);  in bch2_fs_journal_init()
1504 	atomic64_set(&j->reservations.counter,  in bch2_fs_journal_init()
1508 	if (!(init_fifo(&j->pin, JOURNAL_PIN, GFP_KERNEL)))  in bch2_fs_journal_init()
1509 		return -BCH_ERR_ENOMEM_journal_pin_fifo;  in bch2_fs_journal_init()
1511 	for (unsigned i = 0; i < ARRAY_SIZE(j->buf); i++) {  in bch2_fs_journal_init()
1512 		j->buf[i].buf_size = JOURNAL_ENTRY_SIZE_MIN;  in bch2_fs_journal_init()
1513 		j->buf[i].data = kvmalloc(j->buf[i].buf_size, GFP_KERNEL);  in bch2_fs_journal_init()
1514 		if (!j->buf[i].data)  in bch2_fs_journal_init()
1515 			return -BCH_ERR_ENOMEM_journal_buf;  in bch2_fs_journal_init()
1516 		j->buf[i].idx = i;  in bch2_fs_journal_init()
1519 	j->pin.front = j->pin.back = 1;  in bch2_fs_journal_init()
1521 	j->wq = alloc_workqueue("bcachefs_journal",  in bch2_fs_journal_init()
1523 	if (!j->wq)  in bch2_fs_journal_init()
1524 		return -BCH_ERR_ENOMEM_fs_other_alloc;  in bch2_fs_journal_init()
1542 	u64 nr_writes = j->nr_flush_writes + j->nr_noflush_writes;  in __bch2_journal_debug_to_text()
1546 	out->atomic++;  in __bch2_journal_debug_to_text()
1549 	s = READ_ONCE(j->reservations);  in __bch2_journal_debug_to_text()
1552 	prt_bitflags(out, bch2_journal_flags_strs, j->flags);  in __bch2_journal_debug_to_text()
1554 	prt_printf(out, "dirty journal entries:\t%llu/%llu\n",	fifo_used(&j->pin), j->pin.size);  in __bch2_journal_debug_to_text()
1556 	prt_printf(out, "seq_ondisk:\t%llu\n",			j->seq_ondisk);  in __bch2_journal_debug_to_text()
1558 	prt_printf(out, "last_seq_ondisk:\t%llu\n",		j->last_seq_ondisk);  in __bch2_journal_debug_to_text()
1559 	prt_printf(out, "flushed_seq_ondisk:\t%llu\n",		j->flushed_seq_ondisk);  in __bch2_journal_debug_to_text()
1560 	prt_printf(out, "watermark:\t%s\n",			bch2_watermarks[j->watermark]);  in __bch2_journal_debug_to_text()
1561 	prt_printf(out, "each entry reserved:\t%u\n",		j->entry_u64s_reserved);  in __bch2_journal_debug_to_text()
1562 	prt_printf(out, "nr flush writes:\t%llu\n",		j->nr_flush_writes);  in __bch2_journal_debug_to_text()
1563 	prt_printf(out, "nr noflush writes:\t%llu\n",		j->nr_noflush_writes);  in __bch2_journal_debug_to_text()
1565 	prt_human_readable_u64(out, nr_writes ? div64_u64(j->entry_bytes_written, nr_writes) : 0);  in __bch2_journal_debug_to_text()
1567 	prt_printf(out, "nr direct reclaim:\t%llu\n",		j->nr_direct_reclaim);  in __bch2_journal_debug_to_text()
1568 	prt_printf(out, "nr background reclaim:\t%llu\n",	j->nr_background_reclaim);  in __bch2_journal_debug_to_text()
1569 	prt_printf(out, "reclaim kicked:\t%u\n",		j->reclaim_kicked);  in __bch2_journal_debug_to_text()
1570 	prt_printf(out, "reclaim runs in:\t%u ms\n",		time_after(j->next_reclaim, now)  in __bch2_journal_debug_to_text()
1571 	       ? jiffies_to_msecs(j->next_reclaim - jiffies) : 0);  in __bch2_journal_debug_to_text()
1572 	prt_printf(out, "blocked:\t%u\n",			j->blocked);  in __bch2_journal_debug_to_text()
1573 	prt_printf(out, "current entry sectors:\t%u\n",		j->cur_entry_sectors);  in __bch2_journal_debug_to_text()
1574 	prt_printf(out, "current entry error:\t%s\n",		bch2_journal_errors[j->cur_entry_error]);  in __bch2_journal_debug_to_text()
1588 		prt_printf(out, "%u/%u\n", s.cur_entry_offset, j->cur_entry_u64s);  in __bch2_journal_debug_to_text()
1598 	       j->space[journal_space_discarded].next_entry,  in __bch2_journal_debug_to_text()
1599 	       j->space[journal_space_discarded].total);  in __bch2_journal_debug_to_text()
1601 	       j->space[journal_space_clean_ondisk].next_entry,  in __bch2_journal_debug_to_text()
1602 	       j->space[journal_space_clean_ondisk].total);  in __bch2_journal_debug_to_text()
1604 	       j->space[journal_space_clean].next_entry,  in __bch2_journal_debug_to_text()
1605 	       j->space[journal_space_clean].total);  in __bch2_journal_debug_to_text()
1607 	       j->space[journal_space_total].next_entry,  in __bch2_journal_debug_to_text()
1608 	       j->space[journal_space_total].total);  in __bch2_journal_debug_to_text()
1611 	for_each_member_device_rcu(c, ca, &c->rw_devs[BCH_DATA_journal]) {  in __bch2_journal_debug_to_text()
1612 		if (!ca->mi.durability)  in __bch2_journal_debug_to_text()
1615 		struct journal_device *ja = &ca->journal;  in __bch2_journal_debug_to_text()
1617 		if (!test_bit(ca->dev_idx, c->rw_devs[BCH_DATA_journal].d))  in __bch2_journal_debug_to_text()
1620 		if (!ja->nr)  in __bch2_journal_debug_to_text()
1623 		prt_printf(out, "dev %u:\n",			ca->dev_idx);  in __bch2_journal_debug_to_text()
1624 		prt_printf(out, "durability %u:\n",		ca->mi.durability);  in __bch2_journal_debug_to_text()
1626 		prt_printf(out, "nr\t%u\n",			ja->nr);  in __bch2_journal_debug_to_text()
1627 		prt_printf(out, "bucket size\t%u\n",		ca->mi.bucket_size);  in __bch2_journal_debug_to_text()
1628 …\t%u:%u\n",		bch2_journal_dev_buckets_available(j, ja, journal_space_discarded), ja->sectors_free);  in __bch2_journal_debug_to_text()
1629 		prt_printf(out, "discard_idx\t%u\n",		ja->discard_idx);  in __bch2_journal_debug_to_text()
1630 …prt_printf(out, "dirty_ondisk\t%u (seq %llu)\n",ja->dirty_idx_ondisk,	ja->bucket_seq[ja->dirty_idx…  in __bch2_journal_debug_to_text()
1631 		prt_printf(out, "dirty_idx\t%u (seq %llu)\n",	ja->dirty_idx,		ja->bucket_seq[ja->dirty_idx]);  in __bch2_journal_debug_to_text()
1632 		prt_printf(out, "cur_idx\t%u (seq %llu)\n",	ja->cur_idx,		ja->bucket_seq[ja->cur_idx]);  in __bch2_journal_debug_to_text()
1636 …prt_printf(out, "replicas want %u need %u\n", c->opts.metadata_replicas, c->opts.metadata_replicas…  in __bch2_journal_debug_to_text()
1640 	--out->atomic;  in __bch2_journal_debug_to_text()
1645 	spin_lock(&j->lock);  in bch2_journal_debug_to_text()
1647 	spin_unlock(&j->lock);  in bch2_journal_debug_to_text()