Lines Matching +full:pin +full:- +full:count
1 // SPDX-License-Identifier: GPL-2.0
14 #include "sb-members.h"
27 return ja->discard_idx; in journal_space_from()
29 return ja->dirty_idx_ondisk; in journal_space_from()
31 return ja->dirty_idx; in journal_space_from()
41 if (!ja->nr) in bch2_journal_dev_buckets_available()
44 unsigned available = (journal_space_from(ja, from) - in bch2_journal_dev_buckets_available()
45 ja->cur_idx - 1 + ja->nr) % ja->nr; in bch2_journal_dev_buckets_available()
51 if (available && ja->dirty_idx_ondisk == ja->dirty_idx) in bch2_journal_dev_buckets_available()
52 --available; in bch2_journal_dev_buckets_available()
60 bool low_on_space = j->space[journal_space_clean].total * 4 <= in bch2_journal_set_watermark()
61 j->space[journal_space_total].total; in bch2_journal_set_watermark()
62 bool low_on_pin = fifo_free(&j->pin) < j->pin.size / 4; in bch2_journal_set_watermark()
68 if (track_event_change(&c->times[BCH_TIME_blocked_journal_low_on_space], low_on_space) || in bch2_journal_set_watermark()
69 track_event_change(&c->times[BCH_TIME_blocked_journal_low_on_pin], low_on_pin) || in bch2_journal_set_watermark()
70 track_event_change(&c->times[BCH_TIME_blocked_write_buffer_full], low_on_wb)) in bch2_journal_set_watermark()
73 mod_bit(JOURNAL_space_low, &j->flags, low_on_space || low_on_pin); in bch2_journal_set_watermark()
75 swap(watermark, j->watermark); in bch2_journal_set_watermark()
76 if (watermark > j->watermark) in bch2_journal_set_watermark()
84 struct journal_device *ja = &ca->journal; in journal_dev_space_available()
90 .next_entry = ca->mi.bucket_size, in journal_dev_space_available()
91 .total = ca->mi.bucket_size * ja->nr, in journal_dev_space_available()
95 sectors = ja->sectors_free; in journal_dev_space_available()
99 * until we write it out - thus, account for it here: in journal_dev_space_available()
104 unwritten = j->buf[seq & JOURNAL_BUF_MASK].sectors; in journal_dev_space_available()
110 if (unwritten > ca->mi.bucket_size) in journal_dev_space_available()
119 buckets--; in journal_dev_space_available()
120 sectors = ca->mi.bucket_size; in journal_dev_space_available()
123 sectors -= unwritten; in journal_dev_space_available()
126 if (sectors < ca->mi.bucket_size && buckets) { in journal_dev_space_available()
127 buckets--; in journal_dev_space_available()
128 sectors = ca->mi.bucket_size; in journal_dev_space_available()
133 .total = sectors + buckets * ca->mi.bucket_size, in journal_dev_space_available()
148 for_each_member_device_rcu(c, ca, &c->rw_devs[BCH_DATA_journal]) { in __journal_space_available()
149 if (!ca->journal.nr || in __journal_space_available()
150 !ca->mi.durability) in __journal_space_available()
153 min_bucket_size = min(min_bucket_size, ca->mi.bucket_size); in __journal_space_available()
174 space = dev_space[nr_devs_want - 1]; in __journal_space_available()
183 unsigned max_entry_size = min(j->buf[0].buf_size >> 9, in bch2_journal_space_available()
184 j->buf[1].buf_size >> 9); in bch2_journal_space_available()
189 lockdep_assert_held(&j->lock); in bch2_journal_space_available()
192 for_each_member_device_rcu(c, ca, &c->rw_devs[BCH_DATA_journal]) { in bch2_journal_space_available()
193 struct journal_device *ja = &ca->journal; in bch2_journal_space_available()
195 if (!ja->nr) in bch2_journal_space_available()
198 while (ja->dirty_idx != ja->cur_idx && in bch2_journal_space_available()
199 ja->bucket_seq[ja->dirty_idx] < journal_last_seq(j)) in bch2_journal_space_available()
200 ja->dirty_idx = (ja->dirty_idx + 1) % ja->nr; in bch2_journal_space_available()
202 while (ja->dirty_idx_ondisk != ja->dirty_idx && in bch2_journal_space_available()
203 ja->bucket_seq[ja->dirty_idx_ondisk] < j->last_seq_ondisk) in bch2_journal_space_available()
204 ja->dirty_idx_ondisk = (ja->dirty_idx_ondisk + 1) % ja->nr; in bch2_journal_space_available()
206 if (ja->discard_idx != ja->dirty_idx_ondisk) in bch2_journal_space_available()
209 max_entry_size = min_t(unsigned, max_entry_size, ca->mi.bucket_size); in bch2_journal_space_available()
214 j->can_discard = can_discard; in bch2_journal_space_available()
223 for_each_member_device_rcu(c, ca, &c->rw_devs[BCH_DATA_journal]) in bch2_journal_space_available()
224 prt_printf(&buf, " %s", ca->name); in bch2_journal_space_available()
233 nr_devs_want = min_t(unsigned, nr_online, c->opts.metadata_replicas); in bch2_journal_space_available()
236 j->space[i] = __journal_space_available(j, nr_devs_want, i); in bch2_journal_space_available()
238 clean_ondisk = j->space[journal_space_clean_ondisk].total; in bch2_journal_space_available()
239 clean = j->space[journal_space_clean].total; in bch2_journal_space_available()
240 total = j->space[journal_space_total].total; in bch2_journal_space_available()
242 if (!j->space[journal_space_discarded].next_entry) in bch2_journal_space_available()
245 if ((j->space[journal_space_clean_ondisk].next_entry < in bch2_journal_space_available()
246 j->space[journal_space_clean_ondisk].total) && in bch2_journal_space_available()
247 (clean - clean_ondisk <= total / 8) && in bch2_journal_space_available()
249 set_bit(JOURNAL_may_skip_flush, &j->flags); in bch2_journal_space_available()
251 clear_bit(JOURNAL_may_skip_flush, &j->flags); in bch2_journal_space_available()
255 j->cur_entry_sectors = !ret ? j->space[journal_space_discarded].next_entry : 0; in bch2_journal_space_available()
256 j->cur_entry_error = ret; in bch2_journal_space_available()
262 /* Discards - last part of journal reclaim: */
268 spin_lock(&j->lock); in should_discard_bucket()
269 ret = ja->discard_idx != ja->dirty_idx_ondisk; in should_discard_bucket()
270 spin_unlock(&j->lock); in should_discard_bucket()
276 * Advance ja->discard_idx as long as it points to buckets that are no longer
283 mutex_lock(&j->discard_lock); in bch2_journal_do_discards()
286 struct journal_device *ja = &ca->journal; in bch2_journal_do_discards()
289 if (!c->opts.nochanges && in bch2_journal_do_discards()
290 ca->mi.discard && in bch2_journal_do_discards()
291 bdev_max_discard_sectors(ca->disk_sb.bdev)) in bch2_journal_do_discards()
292 blkdev_issue_discard(ca->disk_sb.bdev, in bch2_journal_do_discards()
294 ja->buckets[ja->discard_idx]), in bch2_journal_do_discards()
295 ca->mi.bucket_size, GFP_NOFS); in bch2_journal_do_discards()
297 spin_lock(&j->lock); in bch2_journal_do_discards()
298 ja->discard_idx = (ja->discard_idx + 1) % ja->nr; in bch2_journal_do_discards()
301 spin_unlock(&j->lock); in bch2_journal_do_discards()
305 mutex_unlock(&j->discard_lock); in bch2_journal_do_discards()
309 * Journal entry pinning - machinery for holding a reference on a given journal
317 lockdep_assert_held(&j->lock); in bch2_journal_reclaim_fast()
323 while (!fifo_empty(&j->pin) && in bch2_journal_reclaim_fast()
324 j->pin.front <= j->seq_ondisk && in bch2_journal_reclaim_fast()
325 !atomic_read(&fifo_peek_front(&j->pin).count)) { in bch2_journal_reclaim_fast()
326 j->pin.front++; in bch2_journal_reclaim_fast()
332 __closure_wake_up(&j->reclaim_flush_wait); in bch2_journal_reclaim_fast()
340 return atomic_dec_and_test(&pin_list->count); in __bch2_journal_pin_put()
346 spin_lock(&j->lock); in bch2_journal_pin_put()
348 spin_unlock(&j->lock); in bch2_journal_pin_put()
353 struct journal_entry_pin *pin) in __journal_pin_drop() argument
357 if (!journal_pin_active(pin)) in __journal_pin_drop()
360 if (j->flush_in_progress == pin) in __journal_pin_drop()
361 j->flush_in_progress_dropped = true; in __journal_pin_drop()
363 pin_list = journal_seq_pin(j, pin->seq); in __journal_pin_drop()
364 pin->seq = 0; in __journal_pin_drop()
365 list_del_init(&pin->list); in __journal_pin_drop()
367 if (j->reclaim_flush_wait.list.first) in __journal_pin_drop()
368 __closure_wake_up(&j->reclaim_flush_wait); in __journal_pin_drop()
374 return atomic_dec_and_test(&pin_list->count) && in __journal_pin_drop()
375 pin_list == &fifo_peek_front(&j->pin); in __journal_pin_drop()
379 struct journal_entry_pin *pin) in bch2_journal_pin_drop() argument
381 spin_lock(&j->lock); in bch2_journal_pin_drop()
382 if (__journal_pin_drop(j, pin)) in bch2_journal_pin_drop()
384 spin_unlock(&j->lock); in bch2_journal_pin_drop()
387 static enum journal_pin_type journal_pin_type(struct journal_entry_pin *pin, in journal_pin_type() argument
393 struct btree *b = container_of(pin, struct btree, writes[idx].journal); in journal_pin_type()
395 return JOURNAL_PIN_TYPE_btree0 - b->c.level; in journal_pin_type()
403 struct journal_entry_pin *pin, in bch2_journal_pin_set_locked() argument
415 atomic_inc(&pin_list->count); in bch2_journal_pin_set_locked()
416 pin->seq = seq; in bch2_journal_pin_set_locked()
417 pin->flush = flush_fn; in bch2_journal_pin_set_locked()
419 if (list_empty(&pin_list->unflushed[type]) && in bch2_journal_pin_set_locked()
420 j->reclaim_flush_wait.list.first) in bch2_journal_pin_set_locked()
421 __closure_wake_up(&j->reclaim_flush_wait); in bch2_journal_pin_set_locked()
423 list_add(&pin->list, &pin_list->unflushed[type]); in bch2_journal_pin_set_locked()
431 spin_lock(&j->lock); in bch2_journal_pin_copy()
433 u64 seq = READ_ONCE(src->seq); in bch2_journal_pin_copy()
438 * the src pin - with the pin dropped, the entry to pin might no in bch2_journal_pin_copy()
442 spin_unlock(&j->lock); in bch2_journal_pin_copy()
459 spin_unlock(&j->lock); in bch2_journal_pin_copy()
463 struct journal_entry_pin *pin, in bch2_journal_pin_set() argument
466 spin_lock(&j->lock); in bch2_journal_pin_set()
470 bool reclaim = __journal_pin_drop(j, pin); in bch2_journal_pin_set()
472 bch2_journal_pin_set_locked(j, seq, pin, flush_fn, journal_pin_type(pin, flush_fn)); in bch2_journal_pin_set()
483 spin_unlock(&j->lock); in bch2_journal_pin_set()
487 * bch2_journal_pin_flush: ensure journal pin callback is no longer running
489 * @pin: pin to flush
491 void bch2_journal_pin_flush(struct journal *j, struct journal_entry_pin *pin) in bch2_journal_pin_flush() argument
493 BUG_ON(journal_pin_active(pin)); in bch2_journal_pin_flush()
495 wait_event(j->pin_flush_wait, j->flush_in_progress != pin); in bch2_journal_pin_flush()
517 fifo_for_each_entry_ptr(pin_list, &j->pin, *seq) { in journal_get_next_pin()
524 ret = list_first_entry_or_null(&pin_list->unflushed[i], in journal_get_next_pin()
542 struct journal_entry_pin *pin; in journal_flush_pins() local
548 lockdep_assert_held(&j->reclaim_lock); in journal_flush_pins()
566 j->last_flushed = jiffies; in journal_flush_pins()
568 spin_lock(&j->lock); in journal_flush_pins()
569 pin = journal_get_next_pin(j, seq_to_flush, in journal_flush_pins()
572 if (pin) { in journal_flush_pins()
573 BUG_ON(j->flush_in_progress); in journal_flush_pins()
574 j->flush_in_progress = pin; in journal_flush_pins()
575 j->flush_in_progress_dropped = false; in journal_flush_pins()
576 flush_fn = pin->flush; in journal_flush_pins()
578 spin_unlock(&j->lock); in journal_flush_pins()
580 if (!pin) in journal_flush_pins()
583 if (min_key_cache && pin->flush == bch2_btree_key_cache_journal_flush) in journal_flush_pins()
584 min_key_cache--; in journal_flush_pins()
587 min_any--; in journal_flush_pins()
589 err = flush_fn(j, pin, seq); in journal_flush_pins()
591 spin_lock(&j->lock); in journal_flush_pins()
592 /* Pin might have been dropped or rearmed: */ in journal_flush_pins()
593 if (likely(!err && !j->flush_in_progress_dropped)) in journal_flush_pins()
594 list_move(&pin->list, &journal_seq_pin(j, seq)->flushed[journal_pin_type(pin, flush_fn)]); in journal_flush_pins()
595 j->flush_in_progress = NULL; in journal_flush_pins()
596 j->flush_in_progress_dropped = false; in journal_flush_pins()
597 spin_unlock(&j->lock); in journal_flush_pins()
599 wake_up(&j->pin_flush_wait); in journal_flush_pins()
615 spin_lock(&j->lock); in journal_seq_to_flush()
618 struct journal_device *ja = &ca->journal; in journal_seq_to_flush()
621 if (!ja->nr) in journal_seq_to_flush()
625 nr_buckets = ja->nr / 2; in journal_seq_to_flush()
627 nr_buckets = min(nr_buckets, ja->nr); in journal_seq_to_flush()
629 bucket_to_flush = (ja->cur_idx + nr_buckets) % ja->nr; in journal_seq_to_flush()
631 ja->bucket_seq[bucket_to_flush]); in journal_seq_to_flush()
634 /* Also flush if the pin fifo is more than half full */ in journal_seq_to_flush()
636 (s64) journal_cur_seq(j) - in journal_seq_to_flush()
637 (j->pin.size >> 1)); in journal_seq_to_flush()
638 spin_unlock(&j->lock); in journal_seq_to_flush()
644 * __bch2_journal_reclaim - free up journal buckets
648 * Returns: 0 on success, or -EIO if the journal has been shutdown
654 * - FIFO has fewer than 512 entries left
655 * - fewer than 25% journal buckets free
658 * - FIFO has more than 1024 entries left
659 * - more than 50% journal buckets free
668 struct btree_cache *bc = &c->btree_cache; in __bch2_journal_reclaim()
669 bool kthread = (current->flags & PF_KTHREAD) != 0; in __bch2_journal_reclaim()
676 * We can't invoke memory reclaim while holding the reclaim_lock - in __bch2_journal_reclaim()
681 lockdep_assert_held(&j->reclaim_lock); in __bch2_journal_reclaim()
689 ret = -EIO; in __bch2_journal_reclaim()
699 * If it's been longer than j->reclaim_delay_ms since we last flushed, in __bch2_journal_reclaim()
700 * make sure to flush at least one journal pin: in __bch2_journal_reclaim()
702 if (time_after(jiffies, j->last_flushed + in __bch2_journal_reclaim()
703 msecs_to_jiffies(c->opts.journal_reclaim_delay))) in __bch2_journal_reclaim()
706 if (j->watermark != BCH_WATERMARK_stripe) in __bch2_journal_reclaim()
709 size_t btree_cache_live = bc->live[0].nr + bc->live[1].nr; in __bch2_journal_reclaim()
710 if (atomic_long_read(&bc->nr_dirty) * 2 > btree_cache_live) in __bch2_journal_reclaim()
718 atomic_long_read(&bc->nr_dirty), btree_cache_live, in __bch2_journal_reclaim()
719 atomic_long_read(&c->btree_key_cache.nr_dirty), in __bch2_journal_reclaim()
720 atomic_long_read(&c->btree_key_cache.nr_keys)); in __bch2_journal_reclaim()
727 j->nr_direct_reclaim += nr_flushed; in __bch2_journal_reclaim()
729 j->nr_background_reclaim += nr_flushed; in __bch2_journal_reclaim()
733 wake_up(&j->reclaim_wait); in __bch2_journal_reclaim()
756 j->last_flushed = jiffies; in bch2_journal_reclaim_thread()
759 bool kicked = j->reclaim_kicked; in bch2_journal_reclaim_thread()
761 j->reclaim_kicked = false; in bch2_journal_reclaim_thread()
763 mutex_lock(&j->reclaim_lock); in bch2_journal_reclaim_thread()
765 mutex_unlock(&j->reclaim_lock); in bch2_journal_reclaim_thread()
768 delay = msecs_to_jiffies(c->opts.journal_reclaim_delay); in bch2_journal_reclaim_thread()
769 j->next_reclaim = j->last_flushed + delay; in bch2_journal_reclaim_thread()
771 if (!time_in_range(j->next_reclaim, now, now + delay)) in bch2_journal_reclaim_thread()
772 j->next_reclaim = now + delay; in bch2_journal_reclaim_thread()
778 if (j->reclaim_kicked) in bch2_journal_reclaim_thread()
781 spin_lock(&j->lock); in bch2_journal_reclaim_thread()
782 journal_empty = fifo_empty(&j->pin); in bch2_journal_reclaim_thread()
783 spin_unlock(&j->lock); in bch2_journal_reclaim_thread()
785 long timeout = j->next_reclaim - jiffies; in bch2_journal_reclaim_thread()
802 struct task_struct *p = j->reclaim_thread; in bch2_journal_reclaim_stop()
804 j->reclaim_thread = NULL; in bch2_journal_reclaim_stop()
818 if (j->reclaim_thread) in bch2_journal_reclaim_start()
822 "bch-reclaim/%s", c->name); in bch2_journal_reclaim_start()
829 j->reclaim_thread = p; in bch2_journal_reclaim_start()
840 spin_lock(&j->lock); in journal_pins_still_flushing()
841 fifo_for_each_entry_ptr(pin_list, &j->pin, seq) { in journal_pins_still_flushing()
847 (!list_empty(&pin_list->unflushed[i]) || in journal_pins_still_flushing()
848 !list_empty(&pin_list->flushed[i]))) { in journal_pins_still_flushing()
849 spin_unlock(&j->lock); in journal_pins_still_flushing()
853 spin_unlock(&j->lock); in journal_pins_still_flushing()
874 mutex_lock(&j->reclaim_lock); in journal_flush_done()
876 for (int type = JOURNAL_PIN_TYPE_NR - 1; in journal_flush_done()
878 --type) in journal_flush_done()
887 spin_lock(&j->lock); in journal_flush_done()
892 ret = !test_bit(JOURNAL_replay_done, &j->flags) || in journal_flush_done()
894 !fifo_used(&j->pin); in journal_flush_done()
896 spin_unlock(&j->lock); in journal_flush_done()
898 mutex_unlock(&j->reclaim_lock); in journal_flush_done()
908 if (!test_bit(JOURNAL_running, &j->flags)) in bch2_journal_flush_pins()
911 closure_wait_event(&j->reclaim_flush_wait, in bch2_journal_flush_pins()
924 spin_lock(&j->lock); in bch2_journal_flush_device_pins()
925 fifo_for_each_entry_ptr(p, &j->pin, iter) in bch2_journal_flush_device_pins()
927 ? bch2_dev_list_has_dev(p->devs, dev_idx) in bch2_journal_flush_device_pins()
928 : p->devs.nr < c->opts.metadata_replicas) in bch2_journal_flush_device_pins()
930 spin_unlock(&j->lock); in bch2_journal_flush_device_pins()
938 mutex_lock(&c->replicas_gc_lock); in bch2_journal_flush_device_pins()
948 ret = bch2_journal_meta(&c->journal); in bch2_journal_flush_device_pins()
953 spin_lock(&j->lock); in bch2_journal_flush_device_pins()
958 if (seq >= j->pin.back) in bch2_journal_flush_device_pins()
961 journal_seq_pin(j, seq)->devs); in bch2_journal_flush_device_pins()
965 spin_unlock(&j->lock); in bch2_journal_flush_device_pins()
967 spin_lock(&j->lock); in bch2_journal_flush_device_pins()
970 spin_unlock(&j->lock); in bch2_journal_flush_device_pins()
973 mutex_unlock(&c->replicas_gc_lock); in bch2_journal_flush_device_pins()
981 struct journal_entry_pin *pin; in bch2_journal_seq_pins_to_text() local
983 spin_lock(&j->lock); in bch2_journal_seq_pins_to_text()
984 if (!test_bit(JOURNAL_running, &j->flags)) { in bch2_journal_seq_pins_to_text()
985 spin_unlock(&j->lock); in bch2_journal_seq_pins_to_text()
989 *seq = max(*seq, j->pin.front); in bch2_journal_seq_pins_to_text()
991 if (*seq >= j->pin.back) { in bch2_journal_seq_pins_to_text()
992 spin_unlock(&j->lock); in bch2_journal_seq_pins_to_text()
996 out->atomic++; in bch2_journal_seq_pins_to_text()
1000 prt_printf(out, "%llu: count %u\n", *seq, atomic_read(&pin_list->count)); in bch2_journal_seq_pins_to_text()
1004 for (unsigned i = 0; i < ARRAY_SIZE(pin_list->unflushed); i++) in bch2_journal_seq_pins_to_text()
1005 list_for_each_entry(pin, &pin_list->unflushed[i], list) in bch2_journal_seq_pins_to_text()
1006 prt_printf(out, "\t%px %ps\n", pin, pin->flush); in bch2_journal_seq_pins_to_text()
1009 for (unsigned i = 0; i < ARRAY_SIZE(pin_list->flushed); i++) in bch2_journal_seq_pins_to_text()
1010 list_for_each_entry(pin, &pin_list->flushed[i], list) in bch2_journal_seq_pins_to_text()
1011 prt_printf(out, "\t%px %ps\n", pin, pin->flush); in bch2_journal_seq_pins_to_text()
1015 --out->atomic; in bch2_journal_seq_pins_to_text()
1016 spin_unlock(&j->lock); in bch2_journal_seq_pins_to_text()