52 			       struct journal_replay *j)  in bch2_journal_ptrs_to_text()  argument
54 	darray_for_each(j->ptrs, i) {  in bch2_journal_ptrs_to_text()
55 		if (i != j->ptrs.data)  in bch2_journal_ptrs_to_text()
63 					struct journal_replay *j)  in bch2_journal_replay_to_text()  argument
65 	prt_printf(out, "seq %llu ", le64_to_cpu(j->j.seq));  in bch2_journal_replay_to_text()
67 	bch2_journal_ptrs_to_text(out, c, j);  in bch2_journal_replay_to_text()
69 	for_each_jset_entry_type(entry, &j->j, BCH_JSET_ENTRY_datetime) {  in bch2_journal_replay_to_text()
87 static bool jset_csum_good(struct bch_fs *c, struct jset *j, struct bch_csum *csum)  in jset_csum_good()  argument
89 	if (!bch2_checksum_type_valid(c, JSET_CSUM_TYPE(j))) {  in jset_csum_good()
94 	*csum = csum_vstruct(c, JSET_CSUM_TYPE(j), journal_nonce(j), j);  in jset_csum_good()
95 	return !bch2_crc_cmp(j->csum, *csum);  in jset_csum_good()
108 			     journal_entry_radix_idx(c, le64_to_cpu(i->j.seq)));  in __journal_replay_free()
142 			     struct journal_list *jlist, struct jset *j)  in journal_entry_add()  argument
146 	size_t bytes = vstruct_bytes(j);  in journal_entry_add()
147 	u64 last_seq = !JSET_NO_FLUSH(j) ? le64_to_cpu(j->last_seq) : 0;  in journal_entry_add()
152 	    le64_to_cpu(j->seq) < c->journal.oldest_seq_found_ondisk)  in journal_entry_add()
153 		c->journal.oldest_seq_found_ondisk = le64_to_cpu(j->seq);  in journal_entry_add()
157 	    le64_to_cpu(j->seq) < jlist->last_seq)  in journal_entry_add()
166 		c->journal_entries_base_seq = max_t(s64, 1, le64_to_cpu(j->seq) - S32_MAX);  in journal_entry_add()
177 			if (le64_to_cpu(i->j.seq) >= last_seq)  in journal_entry_add()
187 				journal_entry_radix_idx(c, le64_to_cpu(j->seq)),  in journal_entry_add()
198 		bool identical = bytes == vstruct_bytes(&dup->j) &&  in journal_entry_add()
199 			!memcmp(j, &dup->j, bytes);  in journal_entry_add()
231 	i = kvmalloc(offsetof(struct journal_replay, j) + bytes, GFP_KERNEL);  in journal_entry_add()
239 	unsafe_memcpy(&i->j, j, bytes, "embedded variable length struct");  in journal_entry_add()
1017 	struct jset *j = NULL;  in journal_read_bucket()  local
1060 			j = buf->data;  in journal_read_bucket()
1063 		ret = jset_validate_early(c, ca, j, offset,  in journal_read_bucket()
1067 			sectors = vstruct_sectors(j, c->block_bits);  in journal_read_bucket()
1070 			if (vstruct_bytes(j) > buf->size) {  in journal_read_bucket()
1072 							vstruct_bytes(j));  in journal_read_bucket()
1091 		if (le64_to_cpu(j->seq) > ja->highest_seq_found) {  in journal_read_bucket()
1092 			ja->highest_seq_found = le64_to_cpu(j->seq);  in journal_read_bucket()
1104 		if (le64_to_cpu(j->seq) < ja->bucket_seq[bucket])  in journal_read_bucket()
1107 		ja->bucket_seq[bucket] = le64_to_cpu(j->seq);  in journal_read_bucket()
1109 		enum bch_csum_type csum_type = JSET_CSUM_TYPE(j);  in journal_read_bucket()
1111 		csum_good = jset_csum_good(c, j, &csum);  in journal_read_bucket()
1117 					bch2_csum_err_msg(&err, csum_type, j->csum, csum),  in journal_read_bucket()
1121 		ret = bch2_encrypt(c, JSET_CSUM_TYPE(j), journal_nonce(j),  in journal_read_bucket()
1122 			     j->encrypted_start,  in journal_read_bucket()
1123 			     vstruct_end(j) - (void *) j->encrypted_start);  in journal_read_bucket()
1134 					}, jlist, j);  in journal_read_bucket()
1149 		j = ((void *) j) + (sectors << 9);  in journal_read_bucket()
1259 			*blacklist_seq = *start_seq = le64_to_cpu(i->j.seq) + 1;  in bch2_journal_read()
1261 		if (JSET_NO_FLUSH(&i->j)) {  in bch2_journal_read()
1274 			.journal_seq	= le64_to_cpu(i->j.seq),  in bch2_journal_read()
1276 		if (journal_entry_err_on(le64_to_cpu(i->j.last_seq) > le64_to_cpu(i->j.seq),  in bch2_journal_read()
1277 					 c, le32_to_cpu(i->j.version), &i->j, NULL,  in bch2_journal_read()
1280 					 le64_to_cpu(i->j.last_seq),  in bch2_journal_read()
1281 					 le64_to_cpu(i->j.seq)))  in bch2_journal_read()
1282 			i->j.last_seq = i->j.seq;  in bch2_journal_read()
1284 		*last_seq	= le64_to_cpu(i->j.last_seq);  in bch2_journal_read()
1285 		*blacklist_seq	= le64_to_cpu(i->j.seq) + 1;  in bch2_journal_read()
1314 		seq = le64_to_cpu(i->j.seq);  in bch2_journal_read()
1321 			fsck_err_on(!JSET_NO_FLUSH(&i->j), c,  in bch2_journal_read()
1336 		BUG_ON(seq > le64_to_cpu(i->j.seq));  in bch2_journal_read()
1338 		while (seq < le64_to_cpu(i->j.seq)) {  in bch2_journal_read()
1342 			while (seq < le64_to_cpu(i->j.seq) &&  in bch2_journal_read()
1346 			if (seq == le64_to_cpu(i->j.seq))  in bch2_journal_read()
1351 			while (seq < le64_to_cpu(i->j.seq) &&  in bch2_journal_read()
1357 				prt_printf(&buf1, " size %zu", vstruct_sectors(&prev->j, c->block_bits));  in bch2_journal_read()
1396 						   le64_to_cpu(i->j.seq),  in bch2_journal_read()
1402 				    &i->j,  in bch2_journal_read()
1418 		    (le64_to_cpu(i->j.seq) == *last_seq ||  in bch2_journal_read()
1421 			      le64_to_cpu(i->j.seq), buf.buf))) {  in bch2_journal_read()
1435 static void journal_advance_devs_to_next_bucket(struct journal *j,  in journal_advance_devs_to_next_bucket()  argument
1439 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in journal_advance_devs_to_next_bucket()
1450 		    bch2_journal_dev_buckets_available(j, ja,  in journal_advance_devs_to_next_bucket()
1464 static void __journal_write_alloc(struct journal *j,  in __journal_write_alloc()  argument
1471 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in __journal_write_alloc()
1491 		bch2_dev_stripe_increment(ca, &j->wp.stripe);  in __journal_write_alloc()
1515  * @j:		journal object
1520 static int journal_write_alloc(struct journal *j, struct journal_buf *w)  in journal_write_alloc()  argument
1522 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in journal_write_alloc()
1546 	devs_sorted = bch2_dev_alloc_list(c, &j->wp.stripe, &devs);  in journal_write_alloc()
1548 	__journal_write_alloc(j, w, &devs_sorted, sectors, &replicas, replicas_want);  in journal_write_alloc()
1554 		journal_advance_devs_to_next_bucket(j, &devs_sorted, sectors, w->data->seq);  in journal_write_alloc()
1573 static void journal_buf_realloc(struct journal *j, struct journal_buf *buf)  in journal_buf_realloc()  argument
1575 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in journal_buf_realloc()
1577 	/* we aren't holding j->lock: */  in journal_buf_realloc()
1578 	unsigned new_size = READ_ONCE(j->buf_size_want);  in journal_buf_realloc()
1595 	spin_lock(&j->lock);  in journal_buf_realloc()
1598 	spin_unlock(&j->lock);  in journal_buf_realloc()
1603 static inline struct journal_buf *journal_last_unwritten_buf(struct journal *j)  in journal_last_unwritten_buf()  argument
1605 	return j->buf + (journal_last_unwritten_seq(j) & JOURNAL_BUF_MASK);  in journal_last_unwritten_buf()
1611 	struct journal *j = container_of(w, struct journal, buf[w->idx]);  in CLOSURE_CALLBACK()  local
1612 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in CLOSURE_CALLBACK()
1619 			       ? j->flush_write_time  in CLOSURE_CALLBACK()
1620 			       : j->noflush_write_time, j->write_start_time);  in CLOSURE_CALLBACK()
1637 	spin_lock(&j->lock);  in CLOSURE_CALLBACK()
1638 	if (seq >= j->pin.front)  in CLOSURE_CALLBACK()
1639 		journal_seq_pin(j, seq)->devs = w->devs_written;  in CLOSURE_CALLBACK()
1640 	if (err && (!j->err_seq || seq < j->err_seq))  in CLOSURE_CALLBACK()
1641 		j->err_seq	= seq;  in CLOSURE_CALLBACK()
1646 	for (seq = journal_last_unwritten_seq(j);  in CLOSURE_CALLBACK()
1647 	     seq <= journal_cur_seq(j);  in CLOSURE_CALLBACK()
1649 		w = j->buf + (seq & JOURNAL_BUF_MASK);  in CLOSURE_CALLBACK()
1653 		if (!j->err_seq && !JSET_NO_FLUSH(w->data)) {  in CLOSURE_CALLBACK()
1654 			j->flushed_seq_ondisk = seq;  in CLOSURE_CALLBACK()
1655 			j->last_seq_ondisk = w->last_seq;  in CLOSURE_CALLBACK()
1662 		j->seq_ondisk = seq;  in CLOSURE_CALLBACK()
1671 		if (j->watermark != BCH_WATERMARK_stripe)  in CLOSURE_CALLBACK()
1674 		old.v = atomic64_read(&j->reservations.counter);  in CLOSURE_CALLBACK()
1681 		} while (!atomic64_try_cmpxchg(&j->reservations.counter,  in CLOSURE_CALLBACK()
1689 		bch2_journal_reclaim_fast(j);  in CLOSURE_CALLBACK()
1690 		bch2_journal_space_available(j);  in CLOSURE_CALLBACK()
1694 		journal_wake(j);  in CLOSURE_CALLBACK()
1697 	if (journal_last_unwritten_seq(j) == journal_cur_seq(j) &&  in CLOSURE_CALLBACK()
1699 		struct journal_buf *buf = journal_cur_buf(j);  in CLOSURE_CALLBACK()
1707 		mod_delayed_work(j->wq, &j->write_work, max(0L, delta));  in CLOSURE_CALLBACK()
1716 	bch2_journal_do_writes(j);  in CLOSURE_CALLBACK()
1717 	spin_unlock(&j->lock);  in CLOSURE_CALLBACK()
1724 	struct journal *j = &ca->fs->journal;  in journal_write_endio()  local
1725 	struct journal_buf *w = j->buf + jbio->buf_idx;  in journal_write_endio()
1734 		spin_lock_irqsave(&j->err_lock, flags);  in journal_write_endio()
1736 		spin_unlock_irqrestore(&j->err_lock, flags);  in journal_write_endio()
1746 	struct journal *j = container_of(w, struct journal, buf[w->idx]);  in CLOSURE_CALLBACK()  local
1747 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in CLOSURE_CALLBACK()
1785 	continue_at(cl, journal_write_done, j->wq);  in CLOSURE_CALLBACK()
1791 	struct journal *j = container_of(w, struct journal, buf[w->idx]);  in CLOSURE_CALLBACK()  local
1792 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in CLOSURE_CALLBACK()
1794 	if (j->seq_ondisk + 1 != le64_to_cpu(w->data->seq)) {  in CLOSURE_CALLBACK()
1795 		spin_lock(&j->lock);  in CLOSURE_CALLBACK()
1796 		if (j->seq_ondisk + 1 != le64_to_cpu(w->data->seq)) {  in CLOSURE_CALLBACK()
1797 			closure_wait(&j->async_wait, cl);  in CLOSURE_CALLBACK()
1798 			spin_unlock(&j->lock);  in CLOSURE_CALLBACK()
1799 			continue_at(cl, journal_write_preflush, j->wq);  in CLOSURE_CALLBACK()
1802 		spin_unlock(&j->lock);  in CLOSURE_CALLBACK()
1818 		continue_at(cl, journal_write_submit, j->wq);  in CLOSURE_CALLBACK()
1828 static int bch2_journal_write_prep(struct journal *j, struct journal_buf *w)  in bch2_journal_write_prep()  argument
1830 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in bch2_journal_write_prep()
1915 	WARN_ON(u64s > j->entry_u64s_reserved);  in bch2_journal_write_prep()
1925 				    u64s, w->u64s_reserved, j->entry_u64s_reserved);  in bch2_journal_write_prep()
1936 		j->last_empty_seq = seq;  in bch2_journal_write_prep()
1965 static int bch2_journal_write_pick_flush(struct journal *j, struct journal_buf *w)  in bch2_journal_write_pick_flush()  argument
1967 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in bch2_journal_write_pick_flush()
1968 	int error = bch2_journal_error(j);  in bch2_journal_write_pick_flush()
1986 	if (error && test_bit(JOURNAL_need_flush_write, &j->flags))  in bch2_journal_write_pick_flush()
1992 	     time_before(jiffies, j->last_flush_write +  in bch2_journal_write_pick_flush()
1994 	     test_bit(JOURNAL_may_skip_flush, &j->flags))) {  in bch2_journal_write_pick_flush()
2000 		j->nr_noflush_writes++;  in bch2_journal_write_pick_flush()
2003 		j->last_flush_write = jiffies;  in bch2_journal_write_pick_flush()
2004 		j->nr_flush_writes++;  in bch2_journal_write_pick_flush()
2005 		clear_bit(JOURNAL_need_flush_write, &j->flags);  in bch2_journal_write_pick_flush()
2014 	struct journal *j = container_of(w, struct journal, buf[w->idx]);  in CLOSURE_CALLBACK()  local
2015 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in CLOSURE_CALLBACK()
2028 	j->write_start_time = local_clock();  in CLOSURE_CALLBACK()
2030 	spin_lock(&j->lock);  in CLOSURE_CALLBACK()
2034 	ret = bch2_journal_write_pick_flush(j, w);  in CLOSURE_CALLBACK()
2035 	spin_unlock(&j->lock);  in CLOSURE_CALLBACK()
2039 	mutex_lock(&j->buf_lock);  in CLOSURE_CALLBACK()
2040 	journal_buf_realloc(j, w);  in CLOSURE_CALLBACK()
2042 	ret = bch2_journal_write_prep(j, w);  in CLOSURE_CALLBACK()
2043 	mutex_unlock(&j->buf_lock);  in CLOSURE_CALLBACK()
2047 	j->entry_bytes_written += vstruct_bytes(w->data);  in CLOSURE_CALLBACK()
2050 		spin_lock(&j->lock);  in CLOSURE_CALLBACK()
2051 		ret = journal_write_alloc(j, w);  in CLOSURE_CALLBACK()
2052 		if (!ret || !j->can_discard)  in CLOSURE_CALLBACK()
2055 		spin_unlock(&j->lock);  in CLOSURE_CALLBACK()
2056 		bch2_journal_do_discards(j);  in CLOSURE_CALLBACK()
2059 	if (ret && !bch2_journal_error(j)) {  in CLOSURE_CALLBACK()
2067 		__bch2_journal_debug_to_text(&buf, j);  in CLOSURE_CALLBACK()
2068 		spin_unlock(&j->lock);  in CLOSURE_CALLBACK()
2086 	bch2_journal_space_available(j);  in CLOSURE_CALLBACK()
2087 	bch2_journal_do_writes(j);  in CLOSURE_CALLBACK()
2088 	spin_unlock(&j->lock);  in CLOSURE_CALLBACK()
2106 		continue_at(cl, journal_write_preflush, j->wq);  in CLOSURE_CALLBACK()
2108 		continue_at(cl, journal_write_submit, j->wq);  in CLOSURE_CALLBACK()
2111 	continue_at(cl, journal_write_done, j->wq);  in CLOSURE_CALLBACK()
2115 	continue_at(cl, journal_write_done, j->wq);  in CLOSURE_CALLBACK()