Lines Matching full:j
30 static inline bool journal_seq_unwritten(struct journal *j, u64 seq) in journal_seq_unwritten() argument
32 return seq > j->seq_ondisk; in journal_seq_unwritten()
40 static inline unsigned nr_unwritten_journal_entries(struct journal *j) in nr_unwritten_journal_entries() argument
42 return atomic64_read(&j->seq) - j->seq_ondisk; in nr_unwritten_journal_entries()
45 static bool journal_entry_is_open(struct journal *j) in journal_entry_is_open() argument
47 return __journal_entry_is_open(j->reservations); in journal_entry_is_open()
50 static void bch2_journal_buf_to_text(struct printbuf *out, struct journal *j, u64 seq) in bch2_journal_buf_to_text() argument
52 union journal_res_state s = READ_ONCE(j->reservations); in bch2_journal_buf_to_text()
54 struct journal_buf *buf = j->buf + i; in bch2_journal_buf_to_text()
88 static void bch2_journal_bufs_to_text(struct printbuf *out, struct journal *j) in bch2_journal_bufs_to_text() argument
93 for (u64 seq = journal_last_unwritten_seq(j); in bch2_journal_bufs_to_text()
94 seq <= journal_cur_seq(j); in bch2_journal_bufs_to_text()
96 bch2_journal_buf_to_text(out, j, seq); in bch2_journal_bufs_to_text()
97 prt_printf(out, "last buf %s\n", journal_entry_is_open(j) ? "open" : "closed"); in bch2_journal_bufs_to_text()
101 journal_seq_to_buf(struct journal *j, u64 seq) in journal_seq_to_buf() argument
105 EBUG_ON(seq > journal_cur_seq(j)); in journal_seq_to_buf()
107 if (journal_seq_unwritten(j, seq)) { in journal_seq_to_buf()
108 buf = j->buf + (seq & JOURNAL_BUF_MASK); in journal_seq_to_buf()
136 journal_error_check_stuck(struct journal *j, int error, unsigned flags) in journal_error_check_stuck() argument
138 struct bch_fs *c = container_of(j, struct bch_fs, journal); in journal_error_check_stuck()
144 nr_unwritten_journal_entries(j) || in journal_error_check_stuck()
148 spin_lock(&j->lock); in journal_error_check_stuck()
150 if (j->can_discard) { in journal_error_check_stuck()
151 spin_unlock(&j->lock); in journal_error_check_stuck()
162 if (j->err_seq) { in journal_error_check_stuck()
163 spin_unlock(&j->lock); in journal_error_check_stuck()
166 j->err_seq = journal_cur_seq(j); in journal_error_check_stuck()
167 spin_unlock(&j->lock); in journal_error_check_stuck()
171 bch2_journal_debug_to_text(&buf, j); in journal_error_check_stuck()
175 bch2_journal_pins_to_text(&buf, j); in journal_error_check_stuck()
185 void bch2_journal_do_writes(struct journal *j) in bch2_journal_do_writes() argument
187 for (u64 seq = journal_last_unwritten_seq(j); in bch2_journal_do_writes()
188 seq <= journal_cur_seq(j); in bch2_journal_do_writes()
191 struct journal_buf *w = j->buf + idx; in bch2_journal_do_writes()
198 if (!journal_state_count(j->reservations, idx)) { in bch2_journal_do_writes()
200 closure_call(&w->io, bch2_journal_write, j->wq, NULL); in bch2_journal_do_writes()
212 void bch2_journal_buf_put_final(struct journal *j, u64 seq) in bch2_journal_buf_put_final() argument
214 lockdep_assert_held(&j->lock); in bch2_journal_buf_put_final()
216 if (__bch2_journal_pin_put(j, seq)) in bch2_journal_buf_put_final()
217 bch2_journal_reclaim_fast(j); in bch2_journal_buf_put_final()
218 bch2_journal_do_writes(j); in bch2_journal_buf_put_final()
224 wake_up(&j->wait); in bch2_journal_buf_put_final()
233 static void __journal_entry_close(struct journal *j, unsigned closed_val, bool trace) in __journal_entry_close() argument
235 struct bch_fs *c = container_of(j, struct bch_fs, journal); in __journal_entry_close()
236 struct journal_buf *buf = journal_cur_buf(j); in __journal_entry_close()
243 lockdep_assert_held(&j->lock); in __journal_entry_close()
245 old.v = atomic64_read(&j->reservations.counter); in __journal_entry_close()
253 } while (!atomic64_try_cmpxchg(&j->reservations.counter, in __journal_entry_close()
260 old.cur_entry_offset = j->cur_entry_offset_if_blocked; in __journal_entry_close()
301 buf->last_seq = journal_last_seq(j); in __journal_entry_close()
305 cancel_delayed_work(&j->write_work); in __journal_entry_close()
307 bch2_journal_space_available(j); in __journal_entry_close()
309 __bch2_journal_buf_put(j, old.idx, le64_to_cpu(buf->data->seq)); in __journal_entry_close()
312 void bch2_journal_halt(struct journal *j) in bch2_journal_halt() argument
314 spin_lock(&j->lock); in bch2_journal_halt()
315 __journal_entry_close(j, JOURNAL_ENTRY_ERROR_VAL, true); in bch2_journal_halt()
316 if (!j->err_seq) in bch2_journal_halt()
317 j->err_seq = journal_cur_seq(j); in bch2_journal_halt()
318 journal_wake(j); in bch2_journal_halt()
319 spin_unlock(&j->lock); in bch2_journal_halt()
322 void bch2_journal_halt_locked(struct journal *j) in bch2_journal_halt_locked() argument
324 lockdep_assert_held(&j->lock); in bch2_journal_halt_locked()
326 __journal_entry_close(j, JOURNAL_ENTRY_ERROR_VAL, true); in bch2_journal_halt_locked()
327 if (!j->err_seq) in bch2_journal_halt_locked()
328 j->err_seq = journal_cur_seq(j); in bch2_journal_halt_locked()
329 journal_wake(j); in bch2_journal_halt_locked()
332 static bool journal_entry_want_write(struct journal *j) in journal_entry_want_write() argument
334 bool ret = !journal_entry_is_open(j) || in journal_entry_want_write()
335 journal_cur_seq(j) == journal_last_unwritten_seq(j); in journal_entry_want_write()
339 __journal_entry_close(j, JOURNAL_ENTRY_CLOSED_VAL, true); in journal_entry_want_write()
340 else if (nr_unwritten_journal_entries(j)) { in journal_entry_want_write()
341 struct journal_buf *buf = journal_cur_buf(j); in journal_entry_want_write()
352 bool bch2_journal_entry_close(struct journal *j) in bch2_journal_entry_close() argument
356 spin_lock(&j->lock); in bch2_journal_entry_close()
357 ret = journal_entry_want_write(j); in bch2_journal_entry_close()
358 spin_unlock(&j->lock); in bch2_journal_entry_close()
367 static int journal_entry_open(struct journal *j) in journal_entry_open() argument
369 struct bch_fs *c = container_of(j, struct bch_fs, journal); in journal_entry_open()
370 struct journal_buf *buf = j->buf + in journal_entry_open()
371 ((journal_cur_seq(j) + 1) & JOURNAL_BUF_MASK); in journal_entry_open()
375 lockdep_assert_held(&j->lock); in journal_entry_open()
376 BUG_ON(journal_entry_is_open(j)); in journal_entry_open()
379 if (j->blocked) in journal_entry_open()
382 if (j->cur_entry_error) in journal_entry_open()
383 return j->cur_entry_error; in journal_entry_open()
385 if (bch2_journal_error(j)) in journal_entry_open()
388 if (!fifo_free(&j->pin)) in journal_entry_open()
391 if (nr_unwritten_journal_entries(j) == ARRAY_SIZE(j->buf)) in journal_entry_open()
394 if (journal_cur_seq(j) >= JOURNAL_SEQ_MAX) { in journal_entry_open()
401 BUG_ON(!j->cur_entry_sectors); in journal_entry_open()
404 (journal_cur_seq(j) == j->flushed_seq_ondisk in journal_entry_open()
406 : j->last_flush_write) + in journal_entry_open()
409 buf->u64s_reserved = j->entry_u64s_reserved; in journal_entry_open()
410 buf->disk_sectors = j->cur_entry_sectors; in journal_entry_open()
414 journal_entry_overhead(j); in journal_entry_open()
417 if (u64s <= (ssize_t) j->early_journal_entries.nr) in journal_entry_open()
420 if (fifo_empty(&j->pin) && j->reclaim_thread) in journal_entry_open()
421 wake_up_process(j->reclaim_thread); in journal_entry_open()
424 * The fifo_push() needs to happen at the same time as j->seq is in journal_entry_open()
427 atomic64_inc(&j->seq); in journal_entry_open()
428 journal_pin_list_init(fifo_push_ref(&j->pin), 1); in journal_entry_open()
430 BUG_ON(j->pin.back - 1 != atomic64_read(&j->seq)); in journal_entry_open()
432 BUG_ON(j->buf + (journal_cur_seq(j) & JOURNAL_BUF_MASK) != buf); in journal_entry_open()
445 buf->data->seq = cpu_to_le64(journal_cur_seq(j)); in journal_entry_open()
448 if (j->early_journal_entries.nr) { in journal_entry_open()
449 memcpy(buf->data->_data, j->early_journal_entries.data, in journal_entry_open()
450 j->early_journal_entries.nr * sizeof(u64)); in journal_entry_open()
451 le32_add_cpu(&buf->data->u64s, j->early_journal_entries.nr); in journal_entry_open()
457 j->cur_entry_u64s = u64s; in journal_entry_open()
459 old.v = atomic64_read(&j->reservations.counter); in journal_entry_open()
467 BUG_ON(new.idx != (journal_cur_seq(j) & JOURNAL_BUF_MASK)); in journal_entry_open()
473 } while (!atomic64_try_cmpxchg(&j->reservations.counter, in journal_entry_open()
476 if (nr_unwritten_journal_entries(j) == 1) in journal_entry_open()
477 mod_delayed_work(j->wq, in journal_entry_open()
478 &j->write_work, in journal_entry_open()
480 journal_wake(j); in journal_entry_open()
482 if (j->early_journal_entries.nr) in journal_entry_open()
483 darray_exit(&j->early_journal_entries); in journal_entry_open()
487 static bool journal_quiesced(struct journal *j) in journal_quiesced() argument
489 bool ret = atomic64_read(&j->seq) == j->seq_ondisk; in journal_quiesced()
492 bch2_journal_entry_close(j); in journal_quiesced()
496 static void journal_quiesce(struct journal *j) in journal_quiesce() argument
498 wait_event(j->wait, journal_quiesced(j)); in journal_quiesce()
503 struct journal *j = container_of(work, struct journal, write_work.work); in journal_write_work() local
505 spin_lock(&j->lock); in journal_write_work()
506 if (__journal_entry_is_open(j->reservations)) { in journal_write_work()
507 long delta = journal_cur_buf(j)->expires - jiffies; in journal_write_work()
510 mod_delayed_work(j->wq, &j->write_work, delta); in journal_write_work()
512 __journal_entry_close(j, JOURNAL_ENTRY_CLOSED_VAL, true); in journal_write_work()
514 spin_unlock(&j->lock); in journal_write_work()
517 static int __journal_res_get(struct journal *j, struct journal_res *res, in __journal_res_get() argument
520 struct bch_fs *c = container_of(j, struct bch_fs, journal); in __journal_res_get()
525 if (journal_res_get_fast(j, res, flags)) in __journal_res_get()
528 if (bch2_journal_error(j)) in __journal_res_get()
531 if (j->blocked) in __journal_res_get()
534 if ((flags & BCH_WATERMARK_MASK) < j->watermark) { in __journal_res_get()
536 can_discard = j->can_discard; in __journal_res_get()
540 if (nr_unwritten_journal_entries(j) == ARRAY_SIZE(j->buf) && !journal_entry_is_open(j)) { in __journal_res_get()
545 spin_lock(&j->lock); in __journal_res_get()
552 if (journal_res_get_fast(j, res, flags)) { in __journal_res_get()
562 buf = journal_cur_buf(j); in __journal_res_get()
563 if (journal_entry_is_open(j) && in __journal_res_get()
566 j->buf_size_want = max(j->buf_size_want, buf->buf_size << 1); in __journal_res_get()
568 __journal_entry_close(j, JOURNAL_ENTRY_CLOSED_VAL, false); in __journal_res_get()
569 ret = journal_entry_open(j) ?: JOURNAL_ERR_retry; in __journal_res_get()
571 can_discard = j->can_discard; in __journal_res_get()
572 spin_unlock(&j->lock); in __journal_res_get()
579 if (journal_error_check_stuck(j, ret, flags)) in __journal_res_get()
586 prt_printf(&buf, "seq %llu\n", journal_cur_seq(j)); in __journal_res_get()
587 bch2_journal_bufs_to_text(&buf, j); in __journal_res_get()
601 bch2_journal_do_discards(j); in __journal_res_get()
605 if (mutex_trylock(&j->reclaim_lock)) { in __journal_res_get()
606 bch2_journal_reclaim(j); in __journal_res_get()
607 mutex_unlock(&j->reclaim_lock); in __journal_res_get()
636 int bch2_journal_res_get_slowpath(struct journal *j, struct journal_res *res, in bch2_journal_res_get_slowpath() argument
642 if (closure_wait_event_timeout(&j->async_wait, in bch2_journal_res_get_slowpath()
643 (ret = __journal_res_get(j, res, flags)) != -BCH_ERR_journal_res_get_blocked || in bch2_journal_res_get_slowpath()
651 struct bch_fs *c = container_of(j, struct bch_fs, journal); in bch2_journal_res_get_slowpath()
656 if (closure_wait_event_timeout(&j->async_wait, in bch2_journal_res_get_slowpath()
657 (ret = __journal_res_get(j, res, flags)) != -BCH_ERR_journal_res_get_blocked || in bch2_journal_res_get_slowpath()
663 bch2_journal_debug_to_text(&buf, j); in bch2_journal_res_get_slowpath()
668 closure_wait_event(&j->async_wait, in bch2_journal_res_get_slowpath()
669 (ret = __journal_res_get(j, res, flags)) != -BCH_ERR_journal_res_get_blocked || in bch2_journal_res_get_slowpath()
676 void bch2_journal_entry_res_resize(struct journal *j, in bch2_journal_entry_res_resize() argument
683 spin_lock(&j->lock); in bch2_journal_entry_res_resize()
685 j->entry_u64s_reserved += d; in bch2_journal_entry_res_resize()
689 j->cur_entry_u64s = max_t(int, 0, j->cur_entry_u64s - d); in bch2_journal_entry_res_resize()
691 state = READ_ONCE(j->reservations); in bch2_journal_entry_res_resize()
694 state.cur_entry_offset > j->cur_entry_u64s) { in bch2_journal_entry_res_resize()
695 j->cur_entry_u64s += d; in bch2_journal_entry_res_resize()
699 __journal_entry_close(j, JOURNAL_ENTRY_CLOSED_VAL, true); in bch2_journal_entry_res_resize()
701 journal_cur_buf(j)->u64s_reserved += d; in bch2_journal_entry_res_resize()
704 spin_unlock(&j->lock); in bch2_journal_entry_res_resize()
712 * @j: journal object
721 int bch2_journal_flush_seq_async(struct journal *j, u64 seq, in bch2_journal_flush_seq_async() argument
727 if (seq <= j->flushed_seq_ondisk) in bch2_journal_flush_seq_async()
730 spin_lock(&j->lock); in bch2_journal_flush_seq_async()
732 if (WARN_ONCE(seq > journal_cur_seq(j), in bch2_journal_flush_seq_async()
734 seq, journal_cur_seq(j))) in bch2_journal_flush_seq_async()
738 if (j->err_seq && seq >= j->err_seq) { in bch2_journal_flush_seq_async()
743 if (seq <= j->flushed_seq_ondisk) { in bch2_journal_flush_seq_async()
749 seq = max(seq, journal_last_unwritten_seq(j)); in bch2_journal_flush_seq_async()
752 if (seq > journal_cur_seq(j)) { in bch2_journal_flush_seq_async()
755 if (journal_entry_is_open(j)) in bch2_journal_flush_seq_async()
756 __journal_entry_close(j, JOURNAL_ENTRY_CLOSED_VAL, true); in bch2_journal_flush_seq_async()
758 spin_unlock(&j->lock); in bch2_journal_flush_seq_async()
766 ret = bch2_journal_res_get(j, &res, jset_u64s(0), 0, NULL); in bch2_journal_flush_seq_async()
771 buf = journal_seq_to_buf(j, seq); in bch2_journal_flush_seq_async()
782 bch2_journal_res_put(j, &res); in bch2_journal_flush_seq_async()
784 spin_lock(&j->lock); in bch2_journal_flush_seq_async()
792 buf = journal_seq_to_buf(j, seq); in bch2_journal_flush_seq_async()
799 j->flushing_seq = max(j->flushing_seq, seq); in bch2_journal_flush_seq_async()
804 if (seq == journal_cur_seq(j)) in bch2_journal_flush_seq_async()
805 journal_entry_want_write(j); in bch2_journal_flush_seq_async()
807 spin_unlock(&j->lock); in bch2_journal_flush_seq_async()
811 int bch2_journal_flush_seq(struct journal *j, u64 seq, unsigned task_state) in bch2_journal_flush_seq() argument
819 if (seq <= j->flushed_seq_ondisk) in bch2_journal_flush_seq()
822 ret = wait_event_state(j->wait, in bch2_journal_flush_seq()
823 (ret2 = bch2_journal_flush_seq_async(j, seq, NULL)), in bch2_journal_flush_seq()
827 bch2_time_stats_update(j->flush_seq_time, start_time); in bch2_journal_flush_seq()
836 void bch2_journal_flush_async(struct journal *j, struct closure *parent) in bch2_journal_flush_async() argument
838 bch2_journal_flush_seq_async(j, atomic64_read(&j->seq), parent); in bch2_journal_flush_async()
841 int bch2_journal_flush(struct journal *j) in bch2_journal_flush() argument
843 return bch2_journal_flush_seq(j, atomic64_read(&j->seq), TASK_UNINTERRUPTIBLE); in bch2_journal_flush()
851 bool bch2_journal_noflush_seq(struct journal *j, u64 start, u64 end) in bch2_journal_noflush_seq() argument
853 struct bch_fs *c = container_of(j, struct bch_fs, journal); in bch2_journal_noflush_seq()
863 spin_lock(&j->lock); in bch2_journal_noflush_seq()
867 for (unwritten_seq = journal_last_unwritten_seq(j); in bch2_journal_noflush_seq()
870 struct journal_buf *buf = journal_seq_to_buf(j, unwritten_seq); in bch2_journal_noflush_seq()
881 spin_unlock(&j->lock); in bch2_journal_noflush_seq()
885 static int __bch2_journal_meta(struct journal *j) in __bch2_journal_meta() argument
888 int ret = bch2_journal_res_get(j, &res, jset_u64s(0), 0, NULL); in __bch2_journal_meta()
892 struct journal_buf *buf = j->buf + (res.seq & JOURNAL_BUF_MASK); in __bch2_journal_meta()
900 bch2_journal_res_put(j, &res); in __bch2_journal_meta()
902 return bch2_journal_flush_seq(j, res.seq, TASK_UNINTERRUPTIBLE); in __bch2_journal_meta()
905 int bch2_journal_meta(struct journal *j) in bch2_journal_meta() argument
907 struct bch_fs *c = container_of(j, struct bch_fs, journal); in bch2_journal_meta()
912 int ret = __bch2_journal_meta(j); in bch2_journal_meta()
919 void bch2_journal_unblock(struct journal *j) in bch2_journal_unblock() argument
921 spin_lock(&j->lock); in bch2_journal_unblock()
922 if (!--j->blocked && in bch2_journal_unblock()
923 j->cur_entry_offset_if_blocked < JOURNAL_ENTRY_CLOSED_VAL && in bch2_journal_unblock()
924 j->reservations.cur_entry_offset == JOURNAL_ENTRY_BLOCKED_VAL) { in bch2_journal_unblock()
927 old.v = atomic64_read(&j->reservations.counter); in bch2_journal_unblock()
930 new.cur_entry_offset = j->cur_entry_offset_if_blocked; in bch2_journal_unblock()
931 } while (!atomic64_try_cmpxchg(&j->reservations.counter, &old.v, new.v)); in bch2_journal_unblock()
933 spin_unlock(&j->lock); in bch2_journal_unblock()
935 journal_wake(j); in bch2_journal_unblock()
938 static void __bch2_journal_block(struct journal *j) in __bch2_journal_block() argument
940 if (!j->blocked++) { in __bch2_journal_block()
943 old.v = atomic64_read(&j->reservations.counter); in __bch2_journal_block()
945 j->cur_entry_offset_if_blocked = old.cur_entry_offset; in __bch2_journal_block()
947 if (j->cur_entry_offset_if_blocked >= JOURNAL_ENTRY_CLOSED_VAL) in __bch2_journal_block()
952 } while (!atomic64_try_cmpxchg(&j->reservations.counter, &old.v, new.v)); in __bch2_journal_block()
954 journal_cur_buf(j)->data->u64s = cpu_to_le32(old.cur_entry_offset); in __bch2_journal_block()
958 void bch2_journal_block(struct journal *j) in bch2_journal_block() argument
960 spin_lock(&j->lock); in bch2_journal_block()
961 __bch2_journal_block(j); in bch2_journal_block()
962 spin_unlock(&j->lock); in bch2_journal_block()
964 journal_quiesce(j); in bch2_journal_block()
967 static struct journal_buf *__bch2_next_write_buffer_flush_journal_buf(struct journal *j, in __bch2_next_write_buffer_flush_journal_buf() argument
974 mutex_lock(&j->buf_lock); in __bch2_next_write_buffer_flush_journal_buf()
975 spin_lock(&j->lock); in __bch2_next_write_buffer_flush_journal_buf()
976 max_seq = min(max_seq, journal_cur_seq(j)); in __bch2_next_write_buffer_flush_journal_buf()
978 for (u64 seq = journal_last_unwritten_seq(j); in __bch2_next_write_buffer_flush_journal_buf()
982 struct journal_buf *buf = j->buf + idx; in __bch2_next_write_buffer_flush_journal_buf()
986 s.v = atomic64_read_acquire(&j->reservations.counter); in __bch2_next_write_buffer_flush_journal_buf()
988 unsigned open = seq == journal_cur_seq(j) && __journal_entry_is_open(s); in __bch2_next_write_buffer_flush_journal_buf()
991 __bch2_journal_block(j); in __bch2_next_write_buffer_flush_journal_buf()
1002 spin_unlock(&j->lock); in __bch2_next_write_buffer_flush_journal_buf()
1004 mutex_unlock(&j->buf_lock); in __bch2_next_write_buffer_flush_journal_buf()
1008 struct journal_buf *bch2_next_write_buffer_flush_journal_buf(struct journal *j, in bch2_next_write_buffer_flush_journal_buf() argument
1014 wait_event(j->wait, (ret = __bch2_next_write_buffer_flush_journal_buf(j, in bch2_next_write_buffer_flush_journal_buf()
1017 bch2_journal_unblock(j); in bch2_next_write_buffer_flush_journal_buf()
1260 static bool bch2_journal_writing_to_device(struct journal *j, unsigned dev_idx) in bch2_journal_writing_to_device() argument
1265 spin_lock(&j->lock); in bch2_journal_writing_to_device()
1266 for (seq = journal_last_unwritten_seq(j); in bch2_journal_writing_to_device()
1267 seq <= journal_cur_seq(j) && !ret; in bch2_journal_writing_to_device()
1269 struct journal_buf *buf = journal_seq_to_buf(j, seq); in bch2_journal_writing_to_device()
1274 spin_unlock(&j->lock); in bch2_journal_writing_to_device()
1279 void bch2_dev_journal_stop(struct journal *j, struct bch_dev *ca) in bch2_dev_journal_stop() argument
1281 wait_event(j->wait, !bch2_journal_writing_to_device(j, ca->dev_idx)); in bch2_dev_journal_stop()
1284 void bch2_fs_journal_stop(struct journal *j) in bch2_fs_journal_stop() argument
1286 if (!test_bit(JOURNAL_running, &j->flags)) in bch2_fs_journal_stop()
1289 bch2_journal_reclaim_stop(j); in bch2_fs_journal_stop()
1290 bch2_journal_flush_all_pins(j); in bch2_fs_journal_stop()
1292 wait_event(j->wait, bch2_journal_entry_close(j)); in bch2_fs_journal_stop()
1298 __bch2_journal_meta(j); in bch2_fs_journal_stop()
1300 journal_quiesce(j); in bch2_fs_journal_stop()
1301 cancel_delayed_work_sync(&j->write_work); in bch2_fs_journal_stop()
1303 WARN(!bch2_journal_error(j) && in bch2_fs_journal_stop()
1304 test_bit(JOURNAL_replay_done, &j->flags) && in bch2_fs_journal_stop()
1305 j->last_empty_seq != journal_cur_seq(j), in bch2_fs_journal_stop()
1307 journal_cur_seq(j), j->last_empty_seq); in bch2_fs_journal_stop()
1309 if (!bch2_journal_error(j)) in bch2_fs_journal_stop()
1310 clear_bit(JOURNAL_running, &j->flags); in bch2_fs_journal_stop()
1313 int bch2_fs_journal_start(struct journal *j, u64 cur_seq) in bch2_fs_journal_start() argument
1315 struct bch_fs *c = container_of(j, struct bch_fs, journal); in bch2_fs_journal_start()
1333 last_seq = le64_to_cpu(i->j.last_seq); in bch2_fs_journal_start()
1339 if (nr + 1 > j->pin.size) { in bch2_fs_journal_start()
1340 free_fifo(&j->pin); in bch2_fs_journal_start()
1341 init_fifo(&j->pin, roundup_pow_of_two(nr + 1), GFP_KERNEL); in bch2_fs_journal_start()
1342 if (!j->pin.data) { in bch2_fs_journal_start()
1348 j->replay_journal_seq = last_seq; in bch2_fs_journal_start()
1349 j->replay_journal_seq_end = cur_seq; in bch2_fs_journal_start()
1350 j->last_seq_ondisk = last_seq; in bch2_fs_journal_start()
1351 j->flushed_seq_ondisk = cur_seq - 1; in bch2_fs_journal_start()
1352 j->seq_ondisk = cur_seq - 1; in bch2_fs_journal_start()
1353 j->pin.front = last_seq; in bch2_fs_journal_start()
1354 j->pin.back = cur_seq; in bch2_fs_journal_start()
1355 atomic64_set(&j->seq, cur_seq - 1); in bch2_fs_journal_start()
1357 fifo_for_each_entry_ptr(p, &j->pin, seq) in bch2_fs_journal_start()
1366 seq = le64_to_cpu(i->j.seq); in bch2_fs_journal_start()
1372 if (journal_entry_empty(&i->j)) in bch2_fs_journal_start()
1373 j->last_empty_seq = le64_to_cpu(i->j.seq); in bch2_fs_journal_start()
1375 p = journal_seq_pin(j, seq); in bch2_fs_journal_start()
1385 j->last_empty_seq = cur_seq - 1; /* to match j->seq */ in bch2_fs_journal_start()
1387 spin_lock(&j->lock); in bch2_fs_journal_start()
1389 set_bit(JOURNAL_running, &j->flags); in bch2_fs_journal_start()
1390 j->last_flush_write = jiffies; in bch2_fs_journal_start()
1392 j->reservations.idx = j->reservations.unwritten_idx = journal_cur_seq(j); in bch2_fs_journal_start()
1393 j->reservations.unwritten_idx++; in bch2_fs_journal_start()
1395 c->last_bucket_seq_cleanup = journal_cur_seq(j); in bch2_fs_journal_start()
1397 bch2_journal_space_available(j); in bch2_fs_journal_start()
1398 spin_unlock(&j->lock); in bch2_fs_journal_start()
1400 return bch2_journal_reclaim_start(j); in bch2_fs_journal_start()
1465 for (unsigned j = 0; j < le64_to_cpu(journal_buckets_v2->d[i].nr); j++) in bch2_dev_journal_init() local
1467 le64_to_cpu(journal_buckets_v2->d[i].start) + j; in bch2_dev_journal_init()
1476 void bch2_fs_journal_exit(struct journal *j) in bch2_fs_journal_exit() argument
1478 if (j->wq) in bch2_fs_journal_exit()
1479 destroy_workqueue(j->wq); in bch2_fs_journal_exit()
1481 darray_exit(&j->early_journal_entries); in bch2_fs_journal_exit()
1483 for (unsigned i = 0; i < ARRAY_SIZE(j->buf); i++) in bch2_fs_journal_exit()
1484 kvfree(j->buf[i].data); in bch2_fs_journal_exit()
1485 free_fifo(&j->pin); in bch2_fs_journal_exit()
1488 int bch2_fs_journal_init(struct journal *j) in bch2_fs_journal_init() argument
1492 mutex_init(&j->buf_lock); in bch2_fs_journal_init()
1493 spin_lock_init(&j->lock); in bch2_fs_journal_init()
1494 spin_lock_init(&j->err_lock); in bch2_fs_journal_init()
1495 init_waitqueue_head(&j->wait); in bch2_fs_journal_init()
1496 INIT_DELAYED_WORK(&j->write_work, journal_write_work); in bch2_fs_journal_init()
1497 init_waitqueue_head(&j->reclaim_wait); in bch2_fs_journal_init()
1498 init_waitqueue_head(&j->pin_flush_wait); in bch2_fs_journal_init()
1499 mutex_init(&j->reclaim_lock); in bch2_fs_journal_init()
1500 mutex_init(&j->discard_lock); in bch2_fs_journal_init()
1502 lockdep_init_map(&j->res_map, "journal res", &res_key, 0); in bch2_fs_journal_init()
1504 atomic64_set(&j->reservations.counter, in bch2_fs_journal_init()
1508 if (!(init_fifo(&j->pin, JOURNAL_PIN, GFP_KERNEL))) in bch2_fs_journal_init()
1511 for (unsigned i = 0; i < ARRAY_SIZE(j->buf); i++) { in bch2_fs_journal_init()
1512 j->buf[i].buf_size = JOURNAL_ENTRY_SIZE_MIN; in bch2_fs_journal_init()
1513 j->buf[i].data = kvmalloc(j->buf[i].buf_size, GFP_KERNEL); in bch2_fs_journal_init()
1514 if (!j->buf[i].data) in bch2_fs_journal_init()
1516 j->buf[i].idx = i; in bch2_fs_journal_init()
1519 j->pin.front = j->pin.back = 1; in bch2_fs_journal_init()
1521 j->wq = alloc_workqueue("bcachefs_journal", in bch2_fs_journal_init()
1523 if (!j->wq) in bch2_fs_journal_init()
1537 void __bch2_journal_debug_to_text(struct printbuf *out, struct journal *j) in __bch2_journal_debug_to_text() argument
1539 struct bch_fs *c = container_of(j, struct bch_fs, journal); in __bch2_journal_debug_to_text()
1542 u64 nr_writes = j->nr_flush_writes + j->nr_noflush_writes; in __bch2_journal_debug_to_text()
1549 s = READ_ONCE(j->reservations); in __bch2_journal_debug_to_text()
1552 prt_bitflags(out, bch2_journal_flags_strs, j->flags); in __bch2_journal_debug_to_text()
1554 prt_printf(out, "dirty journal entries:\t%llu/%llu\n", fifo_used(&j->pin), j->pin.size); in __bch2_journal_debug_to_text()
1555 prt_printf(out, "seq:\t%llu\n", journal_cur_seq(j)); in __bch2_journal_debug_to_text()
1556 prt_printf(out, "seq_ondisk:\t%llu\n", j->seq_ondisk); in __bch2_journal_debug_to_text()
1557 prt_printf(out, "last_seq:\t%llu\n", journal_last_seq(j)); in __bch2_journal_debug_to_text()
1558 prt_printf(out, "last_seq_ondisk:\t%llu\n", j->last_seq_ondisk); in __bch2_journal_debug_to_text()
1559 prt_printf(out, "flushed_seq_ondisk:\t%llu\n", j->flushed_seq_ondisk); in __bch2_journal_debug_to_text()
1560 prt_printf(out, "watermark:\t%s\n", bch2_watermarks[j->watermark]); in __bch2_journal_debug_to_text()
1561 prt_printf(out, "each entry reserved:\t%u\n", j->entry_u64s_reserved); in __bch2_journal_debug_to_text()
1562 prt_printf(out, "nr flush writes:\t%llu\n", j->nr_flush_writes); in __bch2_journal_debug_to_text()
1563 prt_printf(out, "nr noflush writes:\t%llu\n", j->nr_noflush_writes); in __bch2_journal_debug_to_text()
1565 prt_human_readable_u64(out, nr_writes ? div64_u64(j->entry_bytes_written, nr_writes) : 0); in __bch2_journal_debug_to_text()
1567 prt_printf(out, "nr direct reclaim:\t%llu\n", j->nr_direct_reclaim); in __bch2_journal_debug_to_text()
1568 prt_printf(out, "nr background reclaim:\t%llu\n", j->nr_background_reclaim); in __bch2_journal_debug_to_text()
1569 prt_printf(out, "reclaim kicked:\t%u\n", j->reclaim_kicked); in __bch2_journal_debug_to_text()
1570 prt_printf(out, "reclaim runs in:\t%u ms\n", time_after(j->next_reclaim, now) in __bch2_journal_debug_to_text()
1571 ? jiffies_to_msecs(j->next_reclaim - jiffies) : 0); in __bch2_journal_debug_to_text()
1572 prt_printf(out, "blocked:\t%u\n", j->blocked); in __bch2_journal_debug_to_text()
1573 prt_printf(out, "current entry sectors:\t%u\n", j->cur_entry_sectors); in __bch2_journal_debug_to_text()
1574 prt_printf(out, "current entry error:\t%s\n", bch2_journal_errors[j->cur_entry_error]); in __bch2_journal_debug_to_text()
1588 prt_printf(out, "%u/%u\n", s.cur_entry_offset, j->cur_entry_u64s); in __bch2_journal_debug_to_text()
1593 bch2_journal_bufs_to_text(out, j); in __bch2_journal_debug_to_text()
1598 j->space[journal_space_discarded].next_entry, in __bch2_journal_debug_to_text()
1599 j->space[journal_space_discarded].total); in __bch2_journal_debug_to_text()
1601 j->space[journal_space_clean_ondisk].next_entry, in __bch2_journal_debug_to_text()
1602 j->space[journal_space_clean_ondisk].total); in __bch2_journal_debug_to_text()
1604 j->space[journal_space_clean].next_entry, in __bch2_journal_debug_to_text()
1605 j->space[journal_space_clean].total); in __bch2_journal_debug_to_text()
1607 j->space[journal_space_total].next_entry, in __bch2_journal_debug_to_text()
1608 j->space[journal_space_total].total); in __bch2_journal_debug_to_text()
1628 …prt_printf(out, "available\t%u:%u\n", bch2_journal_dev_buckets_available(j, ja, journal_space_dis… in __bch2_journal_debug_to_text()
1643 void bch2_journal_debug_to_text(struct printbuf *out, struct journal *j) in bch2_journal_debug_to_text() argument
1645 spin_lock(&j->lock); in bch2_journal_debug_to_text()
1646 __bch2_journal_debug_to_text(out, j); in bch2_journal_debug_to_text()
1647 spin_unlock(&j->lock); in bch2_journal_debug_to_text()