Lines Matching +full:no +full:- +full:wp
1 // SPDX-License-Identifier: GPL-2.0
29 #include "super-io.h"
43 ca->io_latency[rw].quantiles.entries[QUANTILE_IDX(1)].m; in bch2_congested_acct()
46 s64 latency_over = io_latency - latency_threshold; in bch2_congested_acct()
51 * latency_threshold - we don't need much accuracy here so don't in bch2_congested_acct()
54 if (atomic_read(&ca->congested) < CONGESTED_MAX) in bch2_congested_acct()
56 max_t(int, ilog2(latency_threshold) - 2, 0), in bch2_congested_acct()
57 &ca->congested); in bch2_congested_acct()
59 ca->congested_last = now; in bch2_congested_acct()
60 } else if (atomic_read(&ca->congested) > 0) { in bch2_congested_acct()
61 atomic_dec(&ca->congested); in bch2_congested_acct()
67 atomic64_t *latency = &ca->cur_latency[rw]; in bch2_latency_acct()
70 ? now - submit_time in bch2_latency_acct()
78 * latency, skip doing the update and atomic operation - most of in bch2_latency_acct()
81 if (abs((int) (old - io_latency)) < (old >> 1) && in bch2_latency_acct()
90 __bch2_time_stats_update(&ca->io_latency[rw].stats, submit_time, now); in bch2_latency_acct()
103 if (bv->bv_page != ZERO_PAGE(0)) in bch2_bio_free_pages_pool()
104 mempool_free(bv->bv_page, &c->bio_bounce_pages); in bch2_bio_free_pages_pool()
105 bio->bi_vcnt = 0; in bch2_bio_free_pages_pool()
115 mutex_lock(&c->bio_bounce_pages_lock); in __bio_alloc_page_pool()
122 page = mempool_alloc(&c->bio_bounce_pages, GFP_NOFS); in __bio_alloc_page_pool()
138 size -= len; in bch2_bio_alloc_pages_pool()
142 mutex_unlock(&c->bio_bounce_pages_lock); in bch2_bio_alloc_pages_pool()
154 struct bch_fs *c = trans->c; in bch2_sum_sector_overwrites()
168 new->k.p, BTREE_ITER_slots, old, ret) { in bch2_sum_sector_overwrites()
169 s64 sectors = min(new->k.p.offset, old.k->p.offset) - in bch2_sum_sector_overwrites()
170 max(bkey_start_offset(&new->k), in bch2_sum_sector_overwrites()
174 (bkey_extent_is_allocation(&new->k) - in bch2_sum_sector_overwrites()
178 *disk_sectors_delta -= new->k.p.snapshot == old.k->p.snapshot in bch2_sum_sector_overwrites()
183 (new->k.p.snapshot != old.k->p.snapshot || in bch2_sum_sector_overwrites()
188 if (bkey_ge(old.k->p, new->k.p)) in bch2_sum_sector_overwrites()
204 * will set bi->journal_seq to the journal sequence number of this in bch2_extent_update_i_size_sectors()
205 * transaction - for fsync. in bch2_extent_update_i_size_sectors()
209 * to be journalled - if we crash, the bi_journal_seq update will be in bch2_extent_update_i_size_sectors()
217 extent_iter->pos.inode, in bch2_extent_update_i_size_sectors()
218 extent_iter->snapshot), in bch2_extent_update_i_size_sectors()
236 if (unlikely(k_mut->k.type != KEY_TYPE_inode_v3)) { in bch2_extent_update_i_size_sectors()
245 if (!(le64_to_cpu(inode->v.bi_flags) & BCH_INODE_i_size_dirty) && in bch2_extent_update_i_size_sectors()
246 new_i_size > le64_to_cpu(inode->v.bi_size)) { in bch2_extent_update_i_size_sectors()
247 inode->v.bi_size = cpu_to_le64(new_i_size); in bch2_extent_update_i_size_sectors()
252 le64_add_cpu(&inode->v.bi_sectors, i_sectors_delta); in bch2_extent_update_i_size_sectors()
256 if (inode->k.p.snapshot != iter.snapshot) { in bch2_extent_update_i_size_sectors()
257 inode->k.p.snapshot = iter.snapshot; in bch2_extent_update_i_size_sectors()
261 ret = bch2_trans_update(trans, &iter, &inode->k_i, in bch2_extent_update_i_size_sectors()
284 * This traverses us the iterator without changing iter->path->pos to in bch2_extent_update()
286 * path already traversed at iter->pos because in bch2_extent_update()
297 next_pos = k->k.p; in bch2_extent_update()
307 disk_sectors_delta > (s64) disk_res->sectors) { in bch2_extent_update()
308 ret = bch2_disk_reservation_add(trans->c, disk_res, in bch2_extent_update()
309 disk_sectors_delta - disk_res->sectors, in bch2_extent_update()
318 * We always have to do an inode update - even when i_size/i_sectors in bch2_extent_update()
319 * aren't changing - for fsync to work properly; fsync relies on in bch2_extent_update()
320 * inode->bi_journal_seq which is updated by the trigger code: in bch2_extent_update()
323 min(k->k.p.offset << 9, new_i_size), in bch2_extent_update()
340 struct bch_fs *c = op->c; in bch2_write_index_default()
342 struct keylist *keys = &op->insert_keys; in bch2_write_index_default()
347 .subvol = op->subvol, in bch2_write_index_default()
348 .inum = k->k.p.inode, in bch2_write_index_default()
363 &sk.k->k.p.snapshot); in bch2_write_index_default()
370 bkey_start_pos(&sk.k->k), in bch2_write_index_default()
373 ret = bch2_bkey_set_needs_rebalance(c, &op->opts, sk.k) ?: in bch2_write_index_default()
375 &op->res, in bch2_write_index_default()
376 op->new_i_size, &op->i_sectors_delta, in bch2_write_index_default()
377 op->flags & BCH_WRITE_CHECK_ENOSPC); in bch2_write_index_default()
385 if (bkey_ge(iter.pos, k->k.p)) in bch2_write_index_default()
386 bch2_keylist_pop_front(&op->insert_keys); in bch2_write_index_default()
402 bch2_inum_offset_err_msg(op->c, out, in __bch2_write_op_error()
403 (subvol_inum) { op->subvol, op->pos.inode, }, in __bch2_write_op_error()
406 op->flags & BCH_WRITE_MOVE ? "(internal move)" : ""); in __bch2_write_op_error()
411 __bch2_write_op_error(out, op, op->pos.offset); in bch2_write_op_error()
418 (subvol_inum) { op->subvol, op->pos.inode, }, in bch2_write_op_error_trans()
421 op->flags & BCH_WRITE_MOVE ? "(internal move)" : ""); in bch2_write_op_error_trans()
432 BUG_ON(c->opts.nochanges); in bch2_submit_wbio_replicas()
436 ? bch2_dev_have_ref(c, ptr->dev) in bch2_submit_wbio_replicas()
437 : bch2_dev_get_ioref(c, ptr->dev, type == BCH_DATA_btree ? READ : WRITE); in bch2_submit_wbio_replicas()
440 n = to_wbio(bio_alloc_clone(NULL, &wbio->bio, GFP_NOFS, &c->replica_set)); in bch2_submit_wbio_replicas()
442 n->bio.bi_end_io = wbio->bio.bi_end_io; in bch2_submit_wbio_replicas()
443 n->bio.bi_private = wbio->bio.bi_private; in bch2_submit_wbio_replicas()
444 n->parent = wbio; in bch2_submit_wbio_replicas()
445 n->split = true; in bch2_submit_wbio_replicas()
446 n->bounce = false; in bch2_submit_wbio_replicas()
447 n->put_bio = true; in bch2_submit_wbio_replicas()
448 n->bio.bi_opf = wbio->bio.bi_opf; in bch2_submit_wbio_replicas()
449 bio_inc_remaining(&wbio->bio); in bch2_submit_wbio_replicas()
452 n->split = false; in bch2_submit_wbio_replicas()
455 n->c = c; in bch2_submit_wbio_replicas()
456 n->dev = ptr->dev; in bch2_submit_wbio_replicas()
457 n->have_ioref = ca != NULL; in bch2_submit_wbio_replicas()
458 n->nocow = nocow; in bch2_submit_wbio_replicas()
459 n->submit_time = local_clock(); in bch2_submit_wbio_replicas()
460 n->inode_offset = bkey_start_offset(&k->k); in bch2_submit_wbio_replicas()
462 n->nocow_bucket = PTR_BUCKET_NR(ca, ptr); in bch2_submit_wbio_replicas()
463 n->bio.bi_iter.bi_sector = ptr->offset; in bch2_submit_wbio_replicas()
465 if (likely(n->have_ioref)) { in bch2_submit_wbio_replicas()
466 this_cpu_add(ca->io_done->sectors[WRITE][type], in bch2_submit_wbio_replicas()
467 bio_sectors(&n->bio)); in bch2_submit_wbio_replicas()
469 bio_set_dev(&n->bio, ca->disk_sb.bdev); in bch2_submit_wbio_replicas()
471 if (type != BCH_DATA_btree && unlikely(c->opts.no_data_io)) { in bch2_submit_wbio_replicas()
472 bio_endio(&n->bio); in bch2_submit_wbio_replicas()
476 submit_bio(&n->bio); in bch2_submit_wbio_replicas()
478 n->bio.bi_status = BLK_STS_REMOVED; in bch2_submit_wbio_replicas()
479 bio_endio(&n->bio); in bch2_submit_wbio_replicas()
489 struct bch_fs *c = op->c; in bch2_write_done()
491 EBUG_ON(op->open_buckets.nr); in bch2_write_done()
493 bch2_time_stats_update(&c->times[BCH_TIME_data_write], op->start_time); in bch2_write_done()
494 bch2_disk_reservation_put(c, &op->res); in bch2_write_done()
496 if (!(op->flags & BCH_WRITE_MOVE)) in bch2_write_done()
498 bch2_keylist_free(&op->insert_keys, op->inline_keys); in bch2_write_done()
500 EBUG_ON(cl->parent); in bch2_write_done()
502 if (op->end_io) in bch2_write_done()
503 op->end_io(op); in bch2_write_done()
508 struct keylist *keys = &op->insert_keys; in bch2_write_drop_io_error_ptrs()
509 struct bkey_i *src, *dst = keys->keys, *n; in bch2_write_drop_io_error_ptrs()
511 for (src = keys->keys; src != keys->top; src = n) { in bch2_write_drop_io_error_ptrs()
514 if (bkey_extent_is_direct_data(&src->k)) { in bch2_write_drop_io_error_ptrs()
516 test_bit(ptr->dev, op->failed.d)); in bch2_write_drop_io_error_ptrs()
519 return -EIO; in bch2_write_drop_io_error_ptrs()
523 memmove_u64s_down(dst, src, src->k.u64s); in bch2_write_drop_io_error_ptrs()
527 keys->top = dst; in bch2_write_drop_io_error_ptrs()
532 * __bch2_write_index - after a write, update index to point to new data
537 struct bch_fs *c = op->c; in __bch2_write_index()
538 struct keylist *keys = &op->insert_keys; in __bch2_write_index()
542 if (unlikely(op->flags & BCH_WRITE_IO_ERROR)) { in __bch2_write_index()
551 ret = !(op->flags & BCH_WRITE_MOVE) in __bch2_write_index()
558 op->written += sectors_start - keylist_sectors(keys); in __bch2_write_index()
561 struct bkey_i *insert = bch2_keylist_front(&op->insert_keys); in __bch2_write_index()
564 __bch2_write_op_error(&buf, op, bkey_start_offset(&insert->k)); in __bch2_write_index()
575 for_each_set_bit(dev, op->failed.d, BCH_SB_MEMBERS_MAX) in __bch2_write_index()
576 bch2_open_bucket_write_error(c, &op->open_buckets, dev); in __bch2_write_index()
578 bch2_open_buckets_put(c, &op->open_buckets); in __bch2_write_index()
581 keys->top = keys->keys; in __bch2_write_index()
582 op->error = ret; in __bch2_write_index()
583 op->flags |= BCH_WRITE_SUBMITTED; in __bch2_write_index()
587 static inline void __wp_update_state(struct write_point *wp, enum write_point_state state) in __wp_update_state() argument
589 if (state != wp->state) { in __wp_update_state()
592 if (wp->last_state_change && in __wp_update_state()
593 time_after64(now, wp->last_state_change)) in __wp_update_state()
594 wp->time[wp->state] += now - wp->last_state_change; in __wp_update_state()
595 wp->state = state; in __wp_update_state()
596 wp->last_state_change = now; in __wp_update_state()
600 static inline void wp_update_state(struct write_point *wp, bool running) in wp_update_state() argument
605 !list_empty(&wp->writes) ? WRITE_POINT_waiting_io in wp_update_state()
608 __wp_update_state(wp, state); in wp_update_state()
614 struct write_point *wp = op->wp; in CLOSURE_CALLBACK() local
618 if ((op->flags & BCH_WRITE_SUBMITTED) && in CLOSURE_CALLBACK()
619 (op->flags & BCH_WRITE_MOVE)) in CLOSURE_CALLBACK()
620 bch2_bio_free_pages_pool(op->c, &op->wbio.bio); in CLOSURE_CALLBACK()
622 spin_lock_irqsave(&wp->writes_lock, flags); in CLOSURE_CALLBACK()
623 if (wp->state == WRITE_POINT_waiting_io) in CLOSURE_CALLBACK()
624 __wp_update_state(wp, WRITE_POINT_waiting_work); in CLOSURE_CALLBACK()
625 list_add_tail(&op->wp_list, &wp->writes); in CLOSURE_CALLBACK()
626 spin_unlock_irqrestore (&wp->writes_lock, flags); in CLOSURE_CALLBACK()
628 queue_work(wq, &wp->index_update_work); in CLOSURE_CALLBACK()
631 static inline void bch2_write_queue(struct bch_write_op *op, struct write_point *wp) in bch2_write_queue() argument
633 op->wp = wp; in bch2_write_queue()
635 if (wp->state == WRITE_POINT_stopped) { in bch2_write_queue()
636 spin_lock_irq(&wp->writes_lock); in bch2_write_queue()
637 __wp_update_state(wp, WRITE_POINT_waiting_io); in bch2_write_queue()
638 spin_unlock_irq(&wp->writes_lock); in bch2_write_queue()
644 struct write_point *wp = in bch2_write_point_do_index_updates() local
649 spin_lock_irq(&wp->writes_lock); in bch2_write_point_do_index_updates()
650 op = list_pop_entry(&wp->writes, struct bch_write_op, wp_list); in bch2_write_point_do_index_updates()
651 wp_update_state(wp, op != NULL); in bch2_write_point_do_index_updates()
652 spin_unlock_irq(&wp->writes_lock); in bch2_write_point_do_index_updates()
657 op->flags |= BCH_WRITE_IN_WORKER; in bch2_write_point_do_index_updates()
661 if (!(op->flags & BCH_WRITE_SUBMITTED)) in bch2_write_point_do_index_updates()
664 bch2_write_done(&op->cl); in bch2_write_point_do_index_updates()
670 struct closure *cl = bio->bi_private; in bch2_write_endio()
673 struct bch_write_bio *parent = wbio->split ? wbio->parent : NULL; in bch2_write_endio()
674 struct bch_fs *c = wbio->c; in bch2_write_endio()
675 struct bch_dev *ca = wbio->have_ioref in bch2_write_endio()
676 ? bch2_dev_have_ref(c, wbio->dev) in bch2_write_endio()
679 if (bch2_dev_inum_io_err_on(bio->bi_status, ca, BCH_MEMBER_ERROR_write, in bch2_write_endio()
680 op->pos.inode, in bch2_write_endio()
681 wbio->inode_offset << 9, in bch2_write_endio()
683 bch2_blk_status_to_str(bio->bi_status))) { in bch2_write_endio()
684 set_bit(wbio->dev, op->failed.d); in bch2_write_endio()
685 op->flags |= BCH_WRITE_IO_ERROR; in bch2_write_endio()
688 if (wbio->nocow) { in bch2_write_endio()
689 bch2_bucket_nocow_unlock(&c->nocow_locks, in bch2_write_endio()
690 POS(ca->dev_idx, wbio->nocow_bucket), in bch2_write_endio()
692 set_bit(wbio->dev, op->devs_need_flush->d); in bch2_write_endio()
695 if (wbio->have_ioref) { in bch2_write_endio()
696 bch2_latency_acct(ca, wbio->submit_time, WRITE); in bch2_write_endio()
697 percpu_ref_put(&ca->io_ref); in bch2_write_endio()
700 if (wbio->bounce) in bch2_write_endio()
703 if (wbio->put_bio) in bch2_write_endio()
707 bio_endio(&parent->bio); in bch2_write_endio()
713 struct write_point *wp, in init_append_extent() argument
719 op->pos.offset += crc.uncompressed_size; in init_append_extent()
721 e = bkey_extent_init(op->insert_keys.top); in init_append_extent()
722 e->k.p = op->pos; in init_append_extent()
723 e->k.size = crc.uncompressed_size; in init_append_extent()
724 e->k.bversion = version; in init_append_extent()
729 bch2_extent_crc_append(&e->k_i, crc); in init_append_extent()
731 bch2_alloc_sectors_append_ptrs_inlined(op->c, wp, &e->k_i, crc.compressed_size, in init_append_extent()
732 op->flags & BCH_WRITE_CACHED); in init_append_extent()
734 bch2_keylist_push(&op->insert_keys); in init_append_extent()
738 struct write_point *wp, in bch2_write_bio_alloc() argument
746 min(wp->sectors_free << 9, src->bi_iter.bi_size); in bch2_write_bio_alloc()
749 ? ((unsigned long) buf & (PAGE_SIZE - 1)) in bch2_write_bio_alloc()
755 GFP_NOFS, &c->bio_write); in bch2_write_bio_alloc()
757 wbio->put_bio = true; in bch2_write_bio_alloc()
759 wbio->bio.bi_opf = src->bi_opf; in bch2_write_bio_alloc()
766 wbio->bounce = true; in bch2_write_bio_alloc()
769 * We can't use mempool for more than c->sb.encoded_extent_max in bch2_write_bio_alloc()
774 c->opts.encoded_extent_max)); in bch2_write_bio_alloc()
776 if (bio->bi_iter.bi_size < output_available) in bch2_write_bio_alloc()
779 output_available - in bch2_write_bio_alloc()
780 bio->bi_iter.bi_size, in bch2_write_bio_alloc()
790 struct bio *bio = &op->wbio.bio; in bch2_write_rechecksum()
796 if (bch2_csum_type_is_encryption(op->crc.csum_type) != in bch2_write_rechecksum()
798 new_csum_type = op->crc.csum_type; in bch2_write_rechecksum()
800 ret = bch2_rechecksum_bio(c, bio, op->version, op->crc, in bch2_write_rechecksum()
802 op->crc.offset, op->crc.live_size, in bch2_write_rechecksum()
807 bio_advance(bio, op->crc.offset << 9); in bch2_write_rechecksum()
808 bio->bi_iter.bi_size = op->crc.live_size << 9; in bch2_write_rechecksum()
809 op->crc = new_crc; in bch2_write_rechecksum()
815 struct bch_fs *c = op->c; in bch2_write_decrypt()
816 struct nonce nonce = extent_nonce(op->version, op->crc); in bch2_write_decrypt()
820 if (!bch2_csum_type_is_encryption(op->crc.csum_type)) in bch2_write_decrypt()
824 * If we need to decrypt data in the write path, we'll no longer be able in bch2_write_decrypt()
826 * it's decrypted - this is the last point we'll be able to reverify the in bch2_write_decrypt()
829 csum = bch2_checksum_bio(c, op->crc.csum_type, nonce, &op->wbio.bio); in bch2_write_decrypt()
830 if (bch2_crc_cmp(op->crc.csum, csum) && !c->opts.no_data_io) in bch2_write_decrypt()
831 return -EIO; in bch2_write_decrypt()
833 ret = bch2_encrypt_bio(c, op->crc.csum_type, nonce, &op->wbio.bio); in bch2_write_decrypt()
834 op->crc.csum_type = 0; in bch2_write_decrypt()
835 op->crc.csum = (struct bch_csum) { 0, 0 }; in bch2_write_decrypt()
844 } bch2_write_prep_encoded_data(struct bch_write_op *op, struct write_point *wp) in bch2_write_prep_encoded_data() argument
846 struct bch_fs *c = op->c; in bch2_write_prep_encoded_data()
847 struct bio *bio = &op->wbio.bio; in bch2_write_prep_encoded_data()
849 if (!(op->flags & BCH_WRITE_DATA_ENCODED)) in bch2_write_prep_encoded_data()
852 BUG_ON(bio_sectors(bio) != op->crc.compressed_size); in bch2_write_prep_encoded_data()
855 if (op->crc.uncompressed_size == op->crc.live_size && in bch2_write_prep_encoded_data()
856 op->crc.uncompressed_size <= c->opts.encoded_extent_max >> 9 && in bch2_write_prep_encoded_data()
857 op->crc.compressed_size <= wp->sectors_free && in bch2_write_prep_encoded_data()
858 (op->crc.compression_type == bch2_compression_opt_to_type(op->compression_opt) || in bch2_write_prep_encoded_data()
859 op->incompressible)) { in bch2_write_prep_encoded_data()
860 if (!crc_is_compressed(op->crc) && in bch2_write_prep_encoded_data()
861 op->csum_type != op->crc.csum_type && in bch2_write_prep_encoded_data()
862 bch2_write_rechecksum(c, op, op->csum_type) && in bch2_write_prep_encoded_data()
863 !c->opts.no_data_io) in bch2_write_prep_encoded_data()
873 if (crc_is_compressed(op->crc)) { in bch2_write_prep_encoded_data()
880 csum = bch2_checksum_bio(c, op->crc.csum_type, in bch2_write_prep_encoded_data()
881 extent_nonce(op->version, op->crc), in bch2_write_prep_encoded_data()
883 if (bch2_crc_cmp(op->crc.csum, csum) && !c->opts.no_data_io) in bch2_write_prep_encoded_data()
891 * No longer have compressed data after this point - data might be in bch2_write_prep_encoded_data()
899 if ((op->crc.live_size != op->crc.uncompressed_size || in bch2_write_prep_encoded_data()
900 op->crc.csum_type != op->csum_type) && in bch2_write_prep_encoded_data()
901 bch2_write_rechecksum(c, op, op->csum_type) && in bch2_write_prep_encoded_data()
902 !c->opts.no_data_io) in bch2_write_prep_encoded_data()
908 if ((op->compression_opt || in bch2_write_prep_encoded_data()
909 bch2_csum_type_is_encryption(op->crc.csum_type) != in bch2_write_prep_encoded_data()
910 bch2_csum_type_is_encryption(op->csum_type)) && in bch2_write_prep_encoded_data()
917 static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp, in bch2_write_extent() argument
920 struct bch_fs *c = op->c; in bch2_write_extent()
921 struct bio *src = &op->wbio.bio, *dst = src; in bch2_write_extent()
931 ec_buf = bch2_writepoint_ec_buf(c, wp); in bch2_write_extent()
933 switch (bch2_write_prep_encoded_data(op, wp)) { in bch2_write_extent()
937 ret = -EIO; in bch2_write_extent()
944 dst = bch2_write_bio_alloc(c, wp, src, in bch2_write_extent()
950 init_append_extent(op, wp, op->version, op->crc); in bch2_write_extent()
955 op->compression_opt || in bch2_write_extent()
956 (op->csum_type && in bch2_write_extent()
957 !(op->flags & BCH_WRITE_PAGES_STABLE)) || in bch2_write_extent()
958 (bch2_csum_type_is_encryption(op->csum_type) && in bch2_write_extent()
959 !(op->flags & BCH_WRITE_PAGES_OWNED))) { in bch2_write_extent()
960 dst = bch2_write_bio_alloc(c, wp, src, in bch2_write_extent()
966 saved_iter = dst->bi_iter; in bch2_write_extent()
970 struct bversion version = op->version; in bch2_write_extent()
974 dst->bi_iter.bi_size < (wp->sectors_free << 9) && in bch2_write_extent()
975 dst->bi_iter.bi_size < c->opts.encoded_extent_max) in bch2_write_extent()
978 BUG_ON(op->compression_opt && in bch2_write_extent()
979 (op->flags & BCH_WRITE_DATA_ENCODED) && in bch2_write_extent()
980 bch2_csum_type_is_encryption(op->crc.csum_type)); in bch2_write_extent()
981 BUG_ON(op->compression_opt && !bounce); in bch2_write_extent()
983 crc.compression_type = op->incompressible in bch2_write_extent()
985 : op->compression_opt in bch2_write_extent()
987 op->compression_opt) in bch2_write_extent()
990 dst_len = min(dst->bi_iter.bi_size, src->bi_iter.bi_size); in bch2_write_extent()
991 dst_len = min_t(unsigned, dst_len, wp->sectors_free << 9); in bch2_write_extent()
993 if (op->csum_type) in bch2_write_extent()
995 c->opts.encoded_extent_max); in bch2_write_extent()
998 swap(dst->bi_iter.bi_size, dst_len); in bch2_write_extent()
1000 swap(dst->bi_iter.bi_size, dst_len); in bch2_write_extent()
1008 if (bch2_csum_type_is_encryption(op->csum_type)) { in bch2_write_extent()
1010 version.lo = atomic64_inc_return(&c->key_version); in bch2_write_extent()
1012 crc.nonce = op->nonce; in bch2_write_extent()
1013 op->nonce += src_len >> 9; in bch2_write_extent()
1017 if ((op->flags & BCH_WRITE_DATA_ENCODED) && in bch2_write_extent()
1019 bch2_csum_type_is_encryption(op->crc.csum_type) == in bch2_write_extent()
1020 bch2_csum_type_is_encryption(op->csum_type)) { in bch2_write_extent()
1026 * existing checksum covers - if we bounced (because we in bch2_write_extent()
1035 if (bch2_rechecksum_bio(c, src, version, op->crc, in bch2_write_extent()
1036 &crc, &op->crc, in bch2_write_extent()
1038 bio_sectors(src) - (src_len >> 9), in bch2_write_extent()
1039 op->csum_type)) in bch2_write_extent()
1042 * rchecksum_bio sets compression_type on crc from op->crc, in bch2_write_extent()
1049 if ((op->flags & BCH_WRITE_DATA_ENCODED) && in bch2_write_extent()
1050 bch2_rechecksum_bio(c, src, version, op->crc, in bch2_write_extent()
1051 NULL, &op->crc, in bch2_write_extent()
1053 bio_sectors(src) - (src_len >> 9), in bch2_write_extent()
1054 op->crc.csum_type)) in bch2_write_extent()
1061 swap(dst->bi_iter.bi_size, dst_len); in bch2_write_extent()
1062 ret = bch2_encrypt_bio(c, op->csum_type, in bch2_write_extent()
1067 crc.csum = bch2_checksum_bio(c, op->csum_type, in bch2_write_extent()
1069 crc.csum_type = op->csum_type; in bch2_write_extent()
1070 swap(dst->bi_iter.bi_size, dst_len); in bch2_write_extent()
1073 init_append_extent(op, wp, version, crc); in bch2_write_extent()
1080 } while (dst->bi_iter.bi_size && in bch2_write_extent()
1081 src->bi_iter.bi_size && in bch2_write_extent()
1082 wp->sectors_free && in bch2_write_extent()
1083 !bch2_keylist_realloc(&op->insert_keys, in bch2_write_extent()
1084 op->inline_keys, in bch2_write_extent()
1085 ARRAY_SIZE(op->inline_keys), in bch2_write_extent()
1088 more = src->bi_iter.bi_size != 0; in bch2_write_extent()
1090 dst->bi_iter = saved_iter; in bch2_write_extent()
1096 GFP_NOFS, &c->bio_write); in bch2_write_extent()
1097 wbio_init(dst)->put_bio = true; in bch2_write_extent()
1099 dst->bi_opf = src->bi_opf; in bch2_write_extent()
1102 dst->bi_iter.bi_size = total_output; in bch2_write_extent()
1115 ret = -EIO; in bch2_write_extent()
1117 if (to_wbio(dst)->bounce) in bch2_write_extent()
1119 if (to_wbio(dst)->put_bio) in bch2_write_extent()
1128 struct bch_fs *c = op->c; in bch2_extent_is_writeable()
1134 if (k.k->type != KEY_TYPE_extent) in bch2_extent_is_writeable()
1150 return replicas >= op->opts.data_replicas; in bch2_extent_is_writeable()
1169 bch2_cut_front(bkey_start_pos(&orig->k), new); in bch2_nocow_write_convert_one_unwritten()
1170 bch2_cut_back(orig->k.p, new); in bch2_nocow_write_convert_one_unwritten()
1174 ptr->unwritten = 0; in bch2_nocow_write_convert_one_unwritten()
1177 * Note that we're not calling bch2_subvol_get_snapshot() in this path - in bch2_nocow_write_convert_one_unwritten()
1179 * that we update the extent that we wrote to - even if a snapshot has in bch2_nocow_write_convert_one_unwritten()
1184 min(new->k.p.offset << 9, new_i_size), 0) ?: in bch2_nocow_write_convert_one_unwritten()
1191 struct bch_fs *c = op->c; in bch2_nocow_write_convert_unwritten()
1194 for_each_keylist_key(&op->insert_keys, orig) { in bch2_nocow_write_convert_unwritten()
1196 bkey_start_pos(&orig->k), orig->k.p, in bch2_nocow_write_convert_unwritten()
1199 bch2_nocow_write_convert_one_unwritten(trans, &iter, orig, k, op->new_i_size); in bch2_nocow_write_convert_unwritten()
1203 struct bkey_i *insert = bch2_keylist_front(&op->insert_keys); in bch2_nocow_write_convert_unwritten()
1206 bch2_write_op_error_trans(trans, &buf, op, bkey_start_offset(&insert->k)); in bch2_nocow_write_convert_unwritten()
1213 op->error = ret; in bch2_nocow_write_convert_unwritten()
1223 if (unlikely(op->flags & BCH_WRITE_IO_ERROR)) { in __bch2_nocow_write_done()
1224 op->error = -EIO; in __bch2_nocow_write_done()
1225 } else if (unlikely(op->flags & BCH_WRITE_CONVERT_UNWRITTEN)) in __bch2_nocow_write_done()
1245 struct bch_fs *c = op->c; in bch2_nocow_write()
1254 if (op->flags & BCH_WRITE_MOVE) in bch2_nocow_write()
1262 ret = bch2_subvolume_get_snapshot(trans, op->subvol, &snapshot); in bch2_nocow_write()
1267 SPOS(op->pos.inode, op->pos.offset, snapshot), in bch2_nocow_write()
1270 struct bio *bio = &op->wbio.bio; in bch2_nocow_write()
1284 if (unlikely(k.k->p.snapshot != snapshot || in bch2_nocow_write()
1288 if (bch2_keylist_realloc(&op->insert_keys, in bch2_nocow_write()
1289 op->inline_keys, in bch2_nocow_write()
1290 ARRAY_SIZE(op->inline_keys), in bch2_nocow_write()
1291 k.k->u64s)) in bch2_nocow_write()
1297 struct bch_dev *ca = bch2_dev_get_ioref(c, ptr->dev, WRITE); in bch2_nocow_write()
1303 bucket_nocow_lock(&c->nocow_locks, bucket_to_u64(b)); in bch2_nocow_write()
1306 /* XXX allocating memory with btree locks held - rare */ in bch2_nocow_write()
1308 .b = b, .gen = ptr->gen, .l = l, in bch2_nocow_write()
1311 if (ptr->unwritten) in bch2_nocow_write()
1312 op->flags |= BCH_WRITE_CONVERT_UNWRITTEN; in bch2_nocow_write()
1316 bkey_reassemble(op->insert_keys.top, k); in bch2_nocow_write()
1319 bch2_cut_front(op->pos, op->insert_keys.top); in bch2_nocow_write()
1320 if (op->flags & BCH_WRITE_CONVERT_UNWRITTEN) in bch2_nocow_write()
1321 bch2_cut_back(POS(op->pos.inode, op->pos.offset + bio_sectors(bio)), op->insert_keys.top); in bch2_nocow_write()
1324 struct bch_dev *ca = bch2_dev_have_ref(c, i->b.inode); in bch2_nocow_write()
1326 __bch2_bucket_nocow_lock(&c->nocow_locks, i->l, in bch2_nocow_write()
1327 bucket_to_u64(i->b), in bch2_nocow_write()
1330 int gen = bucket_gen_get(ca, i->b.offset); in bch2_nocow_write()
1331 stale = gen < 0 ? gen : gen_after(gen, i->gen); in bch2_nocow_write()
1338 bio = &op->wbio.bio; in bch2_nocow_write()
1339 if (k.k->p.offset < op->pos.offset + bio_sectors(bio)) { in bch2_nocow_write()
1340 bio = bio_split(bio, k.k->p.offset - op->pos.offset, in bch2_nocow_write()
1341 GFP_KERNEL, &c->bio_write); in bch2_nocow_write()
1342 wbio_init(bio)->put_bio = true; in bch2_nocow_write()
1343 bio->bi_opf = op->wbio.bio.bi_opf; in bch2_nocow_write()
1345 op->flags |= BCH_WRITE_SUBMITTED; in bch2_nocow_write()
1348 op->pos.offset += bio_sectors(bio); in bch2_nocow_write()
1349 op->written += bio_sectors(bio); in bch2_nocow_write()
1351 bio->bi_end_io = bch2_write_endio; in bch2_nocow_write()
1352 bio->bi_private = &op->cl; in bch2_nocow_write()
1353 bio->bi_opf |= REQ_OP_WRITE; in bch2_nocow_write()
1354 closure_get(&op->cl); in bch2_nocow_write()
1356 op->insert_keys.top, true); in bch2_nocow_write()
1358 bch2_keylist_push(&op->insert_keys); in bch2_nocow_write()
1359 if (op->flags & BCH_WRITE_SUBMITTED) in bch2_nocow_write()
1378 op->error = ret; in bch2_nocow_write()
1379 op->flags |= BCH_WRITE_SUBMITTED; in bch2_nocow_write()
1383 if (!(op->flags & BCH_WRITE_SUBMITTED)) { in bch2_nocow_write()
1384 closure_sync(&op->cl); in bch2_nocow_write()
1386 op->insert_keys.top = op->insert_keys.keys; in bch2_nocow_write()
1387 } else if (op->flags & BCH_WRITE_SYNC) { in bch2_nocow_write()
1388 closure_sync(&op->cl); in bch2_nocow_write()
1389 bch2_nocow_write_done(&op->cl.work); in bch2_nocow_write()
1396 continue_at(&op->cl, bch2_nocow_write_done, index_update_wq(op)); in bch2_nocow_write()
1401 percpu_ref_put(&bch2_dev_have_ref(c, i->b.inode)->io_ref); in bch2_nocow_write()
1407 bch2_bucket_nocow_unlock(&c->nocow_locks, i->b, BUCKET_NOCOW_LOCK_UPDATE); in bch2_nocow_write()
1415 stale_at->b.inode, in bch2_nocow_write()
1417 ret = -EIO; in bch2_nocow_write()
1420 ret = -BCH_ERR_transaction_restart; in bch2_nocow_write()
1429 struct bch_fs *c = op->c; in __bch2_write()
1430 struct write_point *wp = NULL; in __bch2_write() local
1437 if (unlikely(op->opts.nocow && c->opts.nocow_enabled)) { in __bch2_write()
1439 if (op->flags & BCH_WRITE_SUBMITTED) in __bch2_write()
1443 memset(&op->failed, 0, sizeof(op->failed)); in __bch2_write()
1447 unsigned key_to_write_offset = op->insert_keys.top_p - in __bch2_write()
1448 op->insert_keys.keys_p; in __bch2_write()
1451 if (op->open_buckets.nr + op->nr_replicas + 1 > in __bch2_write()
1452 ARRAY_SIZE(op->open_buckets.v)) in __bch2_write()
1455 if (bch2_keylist_realloc(&op->insert_keys, in __bch2_write()
1456 op->inline_keys, in __bch2_write()
1457 ARRAY_SIZE(op->inline_keys), in __bch2_write()
1462 * The copygc thread is now global, which means it's no longer in __bch2_write()
1468 op->target, in __bch2_write()
1469 op->opts.erasure_code && !(op->flags & BCH_WRITE_CACHED), in __bch2_write()
1470 op->write_point, in __bch2_write()
1471 &op->devs_have, in __bch2_write()
1472 op->nr_replicas, in __bch2_write()
1473 op->nr_replicas_required, in __bch2_write()
1474 op->watermark, in __bch2_write()
1475 op->flags, in __bch2_write()
1476 &op->cl, &wp))); in __bch2_write()
1484 EBUG_ON(!wp); in __bch2_write()
1486 bch2_open_bucket_get(c, wp, &op->open_buckets); in __bch2_write()
1487 ret = bch2_write_extent(op, wp, &bio); in __bch2_write()
1489 bch2_alloc_sectors_done_inlined(c, wp); in __bch2_write()
1492 op->flags |= BCH_WRITE_SUBMITTED; in __bch2_write()
1495 if (!(op->flags & BCH_WRITE_ALLOC_NOWAIT)) { in __bch2_write()
1502 op->error = ret; in __bch2_write()
1507 bio->bi_end_io = bch2_write_endio; in __bch2_write()
1508 bio->bi_private = &op->cl; in __bch2_write()
1509 bio->bi_opf |= REQ_OP_WRITE; in __bch2_write()
1511 closure_get(bio->bi_private); in __bch2_write()
1513 key_to_write = (void *) (op->insert_keys.keys_p + in __bch2_write()
1521 * Sync or no? in __bch2_write()
1527 if ((op->flags & BCH_WRITE_SYNC) || in __bch2_write()
1528 (!(op->flags & BCH_WRITE_SUBMITTED) && in __bch2_write()
1529 !(op->flags & BCH_WRITE_IN_WORKER))) { in __bch2_write()
1530 bch2_wait_on_allocator(c, &op->cl); in __bch2_write()
1534 if (!(op->flags & BCH_WRITE_SUBMITTED)) in __bch2_write()
1536 bch2_write_done(&op->cl); in __bch2_write()
1538 bch2_write_queue(op, wp); in __bch2_write()
1539 continue_at(&op->cl, bch2_write_index, NULL); in __bch2_write()
1547 struct bio *bio = &op->wbio.bio; in bch2_write_data_inline()
1553 memset(&op->failed, 0, sizeof(op->failed)); in bch2_write_data_inline()
1555 op->flags |= BCH_WRITE_WROTE_DATA_INLINE; in bch2_write_data_inline()
1556 op->flags |= BCH_WRITE_SUBMITTED; in bch2_write_data_inline()
1558 bch2_check_set_feature(op->c, BCH_FEATURE_inline_data); in bch2_write_data_inline()
1560 ret = bch2_keylist_realloc(&op->insert_keys, op->inline_keys, in bch2_write_data_inline()
1561 ARRAY_SIZE(op->inline_keys), in bch2_write_data_inline()
1564 op->error = ret; in bch2_write_data_inline()
1569 op->pos.offset += sectors; in bch2_write_data_inline()
1571 id = bkey_inline_data_init(op->insert_keys.top); in bch2_write_data_inline()
1572 id->k.p = op->pos; in bch2_write_data_inline()
1573 id->k.bversion = op->version; in bch2_write_data_inline()
1574 id->k.size = sectors; in bch2_write_data_inline()
1576 iter = bio->bi_iter; in bch2_write_data_inline()
1578 memcpy_from_bio(id->v.data, bio, iter); in bch2_write_data_inline()
1581 id->v.data[data_len++] = '\0'; in bch2_write_data_inline()
1582 set_bkey_val_bytes(&id->k, data_len); in bch2_write_data_inline()
1583 bch2_keylist_push(&op->insert_keys); in bch2_write_data_inline()
1587 bch2_write_done(&op->cl); in bch2_write_data_inline()
1591 * bch2_write() - handle a write to a cache device or flash only volume
1592 * @cl: &bch_write_op->cl
1596 * volume - it's also used by the moving garbage collector to compact data in
1604 * If op->discard is true, instead of inserting the data it invalidates the
1605 * region of the cache represented by op->bio and op->inode.
1610 struct bio *bio = &op->wbio.bio; in CLOSURE_CALLBACK()
1611 struct bch_fs *c = op->c; in CLOSURE_CALLBACK()
1614 EBUG_ON(op->cl.parent); in CLOSURE_CALLBACK()
1615 BUG_ON(!op->nr_replicas); in CLOSURE_CALLBACK()
1616 BUG_ON(!op->write_point.v); in CLOSURE_CALLBACK()
1617 BUG_ON(bkey_eq(op->pos, POS_MAX)); in CLOSURE_CALLBACK()
1619 if (op->flags & BCH_WRITE_ONLY_SPECIFIED_DEVS) in CLOSURE_CALLBACK()
1620 op->flags |= BCH_WRITE_ALLOC_NOWAIT; in CLOSURE_CALLBACK()
1622 op->nr_replicas_required = min_t(unsigned, op->nr_replicas_required, op->nr_replicas); in CLOSURE_CALLBACK()
1623 op->start_time = local_clock(); in CLOSURE_CALLBACK()
1624 bch2_keylist_init(&op->insert_keys, op->inline_keys); in CLOSURE_CALLBACK()
1625 wbio_init(bio)->put_bio = false; in CLOSURE_CALLBACK()
1627 if (unlikely(bio->bi_iter.bi_size & (c->opts.block_size - 1))) { in CLOSURE_CALLBACK()
1632 op->error = -EIO; in CLOSURE_CALLBACK()
1636 if (c->opts.nochanges) { in CLOSURE_CALLBACK()
1637 op->error = -BCH_ERR_erofs_no_writes; in CLOSURE_CALLBACK()
1641 if (!(op->flags & BCH_WRITE_MOVE) && in CLOSURE_CALLBACK()
1643 op->error = -BCH_ERR_erofs_no_writes; in CLOSURE_CALLBACK()
1647 this_cpu_add(c->counters[BCH_COUNTER_io_write], bio_sectors(bio)); in CLOSURE_CALLBACK()
1650 data_len = min_t(u64, bio->bi_iter.bi_size, in CLOSURE_CALLBACK()
1651 op->new_i_size - (op->pos.offset << 9)); in CLOSURE_CALLBACK()
1653 if (c->opts.inline_data && in CLOSURE_CALLBACK()
1662 bch2_disk_reservation_put(c, &op->res); in CLOSURE_CALLBACK()
1664 closure_debug_destroy(&op->cl); in CLOSURE_CALLBACK()
1665 if (op->end_io) in CLOSURE_CALLBACK()
1666 op->end_io(op); in CLOSURE_CALLBACK()
1679 bch2_bpos_to_text(out, op->pos); in bch2_write_op_to_text()
1684 bch2_pr_time_units(out, local_clock() - op->start_time); in bch2_write_op_to_text()
1688 prt_bitflags(out, bch2_write_flags, op->flags); in bch2_write_op_to_text()
1691 prt_printf(out, "ref: %u\n", closure_nr_remaining(&op->cl)); in bch2_write_op_to_text()
1698 mempool_exit(&c->bio_bounce_pages); in bch2_fs_io_write_exit()
1699 bioset_exit(&c->replica_set); in bch2_fs_io_write_exit()
1700 bioset_exit(&c->bio_write); in bch2_fs_io_write_exit()
1705 if (bioset_init(&c->bio_write, 1, offsetof(struct bch_write_bio, bio), BIOSET_NEED_BVECS) || in bch2_fs_io_write_init()
1706 bioset_init(&c->replica_set, 4, offsetof(struct bch_write_bio, bio), 0)) in bch2_fs_io_write_init()
1707 return -BCH_ERR_ENOMEM_bio_write_init; in bch2_fs_io_write_init()
1709 if (mempool_init_page_pool(&c->bio_bounce_pages, in bch2_fs_io_write_init()
1711 c->opts.btree_node_size, in bch2_fs_io_write_init()
1712 c->opts.encoded_extent_max) / in bch2_fs_io_write_init()
1714 return -BCH_ERR_ENOMEM_bio_bounce_pages_init; in bch2_fs_io_write_init()