Lines Matching +full:cache +full:- +full:block
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2009-2011 Red Hat, Inc.
10 #include <linux/dm-bufio.h>
12 #include <linux/device-mapper.h>
13 #include <linux/dm-io.h>
65 * dm_buffer->list_mode
71 /*--------------------------------------------------------------*/
99 /*--------------*/
103 lru->cursor = NULL; in lru_init()
104 lru->count = 0; in lru_init()
105 INIT_LIST_HEAD(&lru->iterators); in lru_init()
110 WARN_ON_ONCE(lru->cursor); in lru_destroy()
111 WARN_ON_ONCE(!list_empty(&lru->iterators)); in lru_destroy()
123 atomic_set(&le->referenced, 0); in lru_insert()
125 if (lru->cursor) { in lru_insert()
126 list_add_tail(&le->list, lru->cursor); in lru_insert()
128 INIT_LIST_HEAD(&le->list); in lru_insert()
129 lru->cursor = &le->list; in lru_insert()
131 lru->count++; in lru_insert()
134 /*--------------*/
149 it->lru = lru; in lru_iter_begin()
150 it->stop = lru->cursor ? to_le(lru->cursor->prev) : NULL; in lru_iter_begin()
151 it->e = lru->cursor ? to_le(lru->cursor) : NULL; in lru_iter_begin()
152 list_add(&it->list, &lru->iterators); in lru_iter_begin()
160 list_del(&it->list); in lru_iter_end()
176 while (it->e) { in lru_iter_next()
177 e = it->e; in lru_iter_next()
180 if (it->e == it->stop) in lru_iter_next()
181 it->e = NULL; in lru_iter_next()
183 it->e = to_le(it->e->list.next); in lru_iter_next()
200 list_for_each_entry(it, &lru->iterators, list) { in lru_iter_invalidate()
201 /* Move c->e forwards if necc. */ in lru_iter_invalidate()
202 if (it->e == e) { in lru_iter_invalidate()
203 it->e = to_le(it->e->list.next); in lru_iter_invalidate()
204 if (it->e == e) in lru_iter_invalidate()
205 it->e = NULL; in lru_iter_invalidate()
208 /* Move it->stop backwards if necc. */ in lru_iter_invalidate()
209 if (it->stop == e) { in lru_iter_invalidate()
210 it->stop = to_le(it->stop->list.prev); in lru_iter_invalidate()
211 if (it->stop == e) in lru_iter_invalidate()
212 it->stop = NULL; in lru_iter_invalidate()
217 /*--------------*/
225 if (lru->count == 1) { in lru_remove()
226 lru->cursor = NULL; in lru_remove()
228 if (lru->cursor == &le->list) in lru_remove()
229 lru->cursor = lru->cursor->next; in lru_remove()
230 list_del(&le->list); in lru_remove()
232 lru->count--; in lru_remove()
240 atomic_set(&le->referenced, 1); in lru_reference()
243 /*--------------*/
260 struct list_head *h = lru->cursor; in lru_evict()
270 while (tested < lru->count) { in lru_evict()
273 if (atomic_read(&le->referenced)) { in lru_evict()
274 atomic_set(&le->referenced, 0); in lru_evict()
283 lru->cursor = le->list.next; in lru_evict()
291 lru->cursor = le->list.next; in lru_evict()
296 h = h->next; in lru_evict()
305 /*--------------------------------------------------------------*/
315 * Describes how the block was allocated:
332 sector_t block; member
366 /*--------------------------------------------------------------*/
369 * The buffer cache manages buffers, particularly:
370 * - inc/dec of holder count
371 * - setting the last_accessed field
372 * - maintains clean/dirty state along with lru
373 * - selecting buffers that match predicates
376 * - allocation/freeing of buffers.
377 * - IO
378 * - Eviction or cache sizing.
407 static inline unsigned int cache_index(sector_t block, unsigned int num_locks) in cache_index() argument
409 return dm_hash_locks_index(block, num_locks); in cache_index()
412 static inline void cache_read_lock(struct dm_buffer_cache *bc, sector_t block) in cache_read_lock() argument
414 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) in cache_read_lock()
415 read_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); in cache_read_lock()
417 down_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); in cache_read_lock()
420 static inline void cache_read_unlock(struct dm_buffer_cache *bc, sector_t block) in cache_read_unlock() argument
422 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) in cache_read_unlock()
423 read_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); in cache_read_unlock()
425 up_read(&bc->trees[cache_index(block, bc->num_locks)].u.lock); in cache_read_unlock()
428 static inline void cache_write_lock(struct dm_buffer_cache *bc, sector_t block) in cache_write_lock() argument
430 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) in cache_write_lock()
431 write_lock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); in cache_write_lock()
433 down_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); in cache_write_lock()
436 static inline void cache_write_unlock(struct dm_buffer_cache *bc, sector_t block) in cache_write_unlock() argument
438 if (static_branch_unlikely(&no_sleep_enabled) && bc->no_sleep) in cache_write_unlock()
439 write_unlock_bh(&bc->trees[cache_index(block, bc->num_locks)].u.spinlock); in cache_write_unlock()
441 up_write(&bc->trees[cache_index(block, bc->num_locks)].u.lock); in cache_write_unlock()
449 struct dm_buffer_cache *cache; member
455 static void lh_init(struct lock_history *lh, struct dm_buffer_cache *cache, bool write) in lh_init() argument
457 lh->cache = cache; in lh_init()
458 lh->write = write; in lh_init()
459 lh->no_previous = cache->num_locks; in lh_init()
460 lh->previous = lh->no_previous; in lh_init()
465 if (lh->write) { in __lh_lock()
466 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) in __lh_lock()
467 write_lock_bh(&lh->cache->trees[index].u.spinlock); in __lh_lock()
469 down_write(&lh->cache->trees[index].u.lock); in __lh_lock()
471 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) in __lh_lock()
472 read_lock_bh(&lh->cache->trees[index].u.spinlock); in __lh_lock()
474 down_read(&lh->cache->trees[index].u.lock); in __lh_lock()
480 if (lh->write) { in __lh_unlock()
481 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) in __lh_unlock()
482 write_unlock_bh(&lh->cache->trees[index].u.spinlock); in __lh_unlock()
484 up_write(&lh->cache->trees[index].u.lock); in __lh_unlock()
486 if (static_branch_unlikely(&no_sleep_enabled) && lh->cache->no_sleep) in __lh_unlock()
487 read_unlock_bh(&lh->cache->trees[index].u.spinlock); in __lh_unlock()
489 up_read(&lh->cache->trees[index].u.lock); in __lh_unlock()
498 if (lh->previous != lh->no_previous) { in lh_exit()
499 __lh_unlock(lh, lh->previous); in lh_exit()
500 lh->previous = lh->no_previous; in lh_exit()
510 unsigned int index = cache_index(b, lh->no_previous); /* no_previous is num_locks */ in lh_next()
512 if (lh->previous != lh->no_previous) { in lh_next()
513 if (lh->previous != index) { in lh_next()
514 __lh_unlock(lh, lh->previous); in lh_next()
516 lh->previous = index; in lh_next()
520 lh->previous = index; in lh_next()
540 bc->num_locks = num_locks; in cache_init()
541 bc->no_sleep = no_sleep; in cache_init()
543 for (i = 0; i < bc->num_locks; i++) { in cache_init()
545 rwlock_init(&bc->trees[i].u.spinlock); in cache_init()
547 init_rwsem(&bc->trees[i].u.lock); in cache_init()
548 bc->trees[i].root = RB_ROOT; in cache_init()
551 lru_init(&bc->lru[LIST_CLEAN]); in cache_init()
552 lru_init(&bc->lru[LIST_DIRTY]); in cache_init()
559 for (i = 0; i < bc->num_locks; i++) in cache_destroy()
560 WARN_ON_ONCE(!RB_EMPTY_ROOT(&bc->trees[i].root)); in cache_destroy()
562 lru_destroy(&bc->lru[LIST_CLEAN]); in cache_destroy()
563 lru_destroy(&bc->lru[LIST_DIRTY]); in cache_destroy()
566 /*--------------*/
573 return bc->lru[list_mode].count; in cache_count()
581 /*--------------*/
584 * Gets a specific buffer, indexed by block.
590 static struct dm_buffer *__cache_get(const struct rb_root *root, sector_t block) in __cache_get() argument
592 struct rb_node *n = root->rb_node; in __cache_get()
598 if (b->block == block) in __cache_get()
601 n = block < b->block ? n->rb_left : n->rb_right; in __cache_get()
609 atomic_inc(&b->hold_count); in __cache_inc_buffer()
610 WRITE_ONCE(b->last_accessed, jiffies); in __cache_inc_buffer()
613 static struct dm_buffer *cache_get(struct dm_buffer_cache *bc, sector_t block) in cache_get() argument
617 cache_read_lock(bc, block); in cache_get()
618 b = __cache_get(&bc->trees[cache_index(block, bc->num_locks)].root, block); in cache_get()
620 lru_reference(&b->lru); in cache_get()
623 cache_read_unlock(bc, block); in cache_get()
628 /*--------------*/
638 cache_read_lock(bc, b->block); in cache_put()
639 BUG_ON(!atomic_read(&b->hold_count)); in cache_put()
640 r = atomic_dec_and_test(&b->hold_count); in cache_put()
641 cache_read_unlock(bc, b->block); in cache_put()
646 /*--------------*/
670 lh_next(w->lh, b->block); in __evict_pred()
672 if (atomic_read(&b->hold_count)) in __evict_pred()
675 return w->pred(b, w->context); in __evict_pred()
686 le = lru_evict(&bc->lru[list_mode], __evict_pred, &w, bc->no_sleep); in __cache_evict()
692 rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root); in __cache_evict()
710 /*--------------*/
717 cache_write_lock(bc, b->block); in cache_mark()
718 if (list_mode != b->list_mode) { in cache_mark()
719 lru_remove(&bc->lru[b->list_mode], &b->lru); in cache_mark()
720 b->list_mode = list_mode; in cache_mark()
721 lru_insert(&bc->lru[b->list_mode], &b->lru); in cache_mark()
723 cache_write_unlock(bc, b->block); in cache_mark()
726 /*--------------*/
740 le = lru_evict(&bc->lru[old_mode], __evict_pred, &w, bc->no_sleep); in __cache_mark_many()
745 b->list_mode = new_mode; in __cache_mark_many()
746 lru_insert(&bc->lru[b->list_mode], &b->lru); in __cache_mark_many()
760 /*--------------*/
781 struct lru *lru = &bc->lru[list_mode]; in __cache_iterate()
784 if (!lru->cursor) in __cache_iterate()
787 first = le = to_le(lru->cursor); in __cache_iterate()
791 lh_next(lh, b->block); in __cache_iterate()
802 le = to_le(le->list.next); in __cache_iterate()
816 /*--------------*/
819 * Passes ownership of the buffer to the cache. Returns false if the
829 struct rb_node **new = &root->rb_node, *parent = NULL; in __cache_insert()
835 if (found->block == b->block) in __cache_insert()
839 new = b->block < found->block ? in __cache_insert()
840 &found->node.rb_left : &found->node.rb_right; in __cache_insert()
843 rb_link_node(&b->node, parent, new); in __cache_insert()
844 rb_insert_color(&b->node, root); in __cache_insert()
853 if (WARN_ON_ONCE(b->list_mode >= LIST_SIZE)) in cache_insert()
856 cache_write_lock(bc, b->block); in cache_insert()
857 BUG_ON(atomic_read(&b->hold_count) != 1); in cache_insert()
858 r = __cache_insert(&bc->trees[cache_index(b->block, bc->num_locks)].root, b); in cache_insert()
860 lru_insert(&bc->lru[b->list_mode], &b->lru); in cache_insert()
861 cache_write_unlock(bc, b->block); in cache_insert()
866 /*--------------*/
869 * Removes buffer from cache, ownership of the buffer passes back to the caller.
878 cache_write_lock(bc, b->block); in cache_remove()
880 if (atomic_read(&b->hold_count) != 1) { in cache_remove()
884 rb_erase(&b->node, &bc->trees[cache_index(b->block, bc->num_locks)].root); in cache_remove()
885 lru_remove(&bc->lru[b->list_mode], &b->lru); in cache_remove()
888 cache_write_unlock(bc, b->block); in cache_remove()
893 /*--------------*/
897 static struct dm_buffer *__find_next(struct rb_root *root, sector_t block) in __find_next() argument
899 struct rb_node *n = root->rb_node; in __find_next()
906 if (b->block == block) in __find_next()
909 if (block <= b->block) { in __find_next()
910 n = n->rb_left; in __find_next()
913 n = n->rb_right; in __find_next()
931 if (!b || (b->block >= end)) in __remove_range()
934 begin = b->block + 1; in __remove_range()
936 if (atomic_read(&b->hold_count)) in __remove_range()
940 rb_erase(&b->node, root); in __remove_range()
941 lru_remove(&bc->lru[b->list_mode], &b->lru); in __remove_range()
953 BUG_ON(bc->no_sleep); in cache_remove_range()
954 for (i = 0; i < bc->num_locks; i++) { in cache_remove_range()
955 down_write(&bc->trees[i].u.lock); in cache_remove_range()
956 __remove_range(bc, &bc->trees[i].root, begin, end, pred, release); in cache_remove_range()
957 up_write(&bc->trees[i].u.lock); in cache_remove_range()
961 /*----------------------------------------------------------------*/
974 * context), so some clean-not-writing buffers can be held on
1015 struct dm_buffer_cache cache; /* must be last member */ member
1018 /*----------------------------------------------------------------*/
1020 #define dm_bufio_in_request() (!!current->bio_list)
1024 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep) in dm_bufio_lock()
1025 spin_lock_bh(&c->spinlock); in dm_bufio_lock()
1027 mutex_lock_nested(&c->lock, dm_bufio_in_request()); in dm_bufio_lock()
1032 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep) in dm_bufio_unlock()
1033 spin_unlock_bh(&c->spinlock); in dm_bufio_unlock()
1035 mutex_unlock(&c->lock); in dm_bufio_unlock()
1038 /*----------------------------------------------------------------*/
1041 * Default cache size: available memory divided by the ratio.
1046 * Total cache size set by the user.
1052 * at any time. If it disagrees, the user has changed cache size.
1071 /*----------------------------------------------------------------*/
1096 b->stack_len = stack_trace_save(b->stack_entries, MAX_STACK, 2); in buffer_record_stack()
1100 /*----------------------------------------------------------------*/
1114 data_mode = b->data_mode; in adjust_total_allocated()
1115 diff = (long)b->c->block_size; in adjust_total_allocated()
1117 diff = -diff; in adjust_total_allocated()
1137 * Change the number of clients and recalculate per-client limit.
1149 * Use default if set to 0 and report the actual cache size used. in __cache_size_refresh()
1182 if (unlikely(c->slab_cache != NULL)) { in alloc_buffer_data()
1184 return kmem_cache_alloc(c->slab_cache, gfp_mask); in alloc_buffer_data()
1187 if (unlikely(c->block_size < PAGE_SIZE)) { in alloc_buffer_data()
1189 return kmalloc(c->block_size, gfp_mask | __GFP_RECLAIMABLE); in alloc_buffer_data()
1192 if (c->block_size <= KMALLOC_MAX_SIZE && in alloc_buffer_data()
1196 c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT)); in alloc_buffer_data()
1201 return __vmalloc(c->block_size, gfp_mask); in alloc_buffer_data()
1212 kmem_cache_free(c->slab_cache, data); in free_buffer_data()
1221 c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT)); in free_buffer_data()
1240 struct dm_buffer *b = kmem_cache_alloc(c->slab_buffer, gfp_mask); in alloc_buffer()
1245 b->c = c; in alloc_buffer()
1247 b->data = alloc_buffer_data(c, gfp_mask, &b->data_mode); in alloc_buffer()
1248 if (!b->data) { in alloc_buffer()
1249 kmem_cache_free(c->slab_buffer, b); in alloc_buffer()
1255 b->stack_len = 0; in alloc_buffer()
1265 struct dm_bufio_client *c = b->c; in free_buffer()
1268 free_buffer_data(c, b->data, b->data_mode); in free_buffer()
1269 kmem_cache_free(c->slab_buffer, b); in free_buffer()
1273 *--------------------------------------------------------------------------
1278 * memory-consumption per buffer, so it is not viable);
1280 * the memory must be direct-mapped, not vmalloced;
1286 * rejects the bio because it is too large, use dm-io layer to do the I/O.
1287 * The dm-io layer splits the I/O into multiple requests, avoiding the above
1289 *--------------------------------------------------------------------------
1293 * dm-io completion routine. It just calls b->bio.bi_end_io, pretending
1300 b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : 0); in dmio_complete()
1312 .client = b->c->dm_io, in use_dmio()
1315 .bdev = b->c->bdev, in use_dmio()
1320 if (b->data_mode != DATA_MODE_VMALLOC) { in use_dmio()
1322 io_req.mem.ptr.addr = (char *)b->data + offset; in use_dmio()
1325 io_req.mem.ptr.vma = (char *)b->data + offset; in use_dmio()
1330 b->end_io(b, errno_to_blk_status(r)); in use_dmio()
1335 struct dm_buffer *b = bio->bi_private; in bio_complete()
1336 blk_status_t status = bio->bi_status; in bio_complete()
1340 b->end_io(b, status); in bio_complete()
1356 bio_init(bio, b->c->bdev, bio->bi_inline_vecs, 1, op); in use_bio()
1357 bio->bi_iter.bi_sector = sector; in use_bio()
1358 bio->bi_end_io = bio_complete; in use_bio()
1359 bio->bi_private = b; in use_bio()
1360 bio->bi_ioprio = ioprio; in use_bio()
1362 ptr = (char *)b->data + offset; in use_bio()
1370 static inline sector_t block_to_sector(struct dm_bufio_client *c, sector_t block) in block_to_sector() argument
1374 if (likely(c->sectors_per_block_bits >= 0)) in block_to_sector()
1375 sector = block << c->sectors_per_block_bits; in block_to_sector()
1377 sector = block * (c->block_size >> SECTOR_SHIFT); in block_to_sector()
1378 sector += c->start; in block_to_sector()
1390 b->end_io = end_io; in submit_io()
1392 sector = block_to_sector(b->c, b->block); in submit_io()
1395 n_sectors = b->c->block_size >> SECTOR_SHIFT; in submit_io()
1398 if (b->c->write_callback) in submit_io()
1399 b->c->write_callback(b); in submit_io()
1400 offset = b->write_start; in submit_io()
1401 end = b->write_end; in submit_io()
1402 offset &= -DM_BUFIO_WRITE_ALIGN; in submit_io()
1403 end += DM_BUFIO_WRITE_ALIGN - 1; in submit_io()
1404 end &= -DM_BUFIO_WRITE_ALIGN; in submit_io()
1405 if (unlikely(end > b->c->block_size)) in submit_io()
1406 end = b->c->block_size; in submit_io()
1409 n_sectors = (end - offset) >> SECTOR_SHIFT; in submit_io()
1412 if (b->data_mode != DATA_MODE_VMALLOC) in submit_io()
1419 *--------------------------------------------------------------
1421 *--------------------------------------------------------------
1432 b->write_error = status; in write_endio()
1434 struct dm_bufio_client *c = b->c; in write_endio()
1436 (void)cmpxchg(&c->async_write_error, 0, in write_endio()
1440 BUG_ON(!test_bit(B_WRITING, &b->state)); in write_endio()
1443 clear_bit(B_WRITING, &b->state); in write_endio()
1446 wake_up_bit(&b->state, B_WRITING); in write_endio()
1452 * - If the buffer is not dirty, exit.
1453 * - If there some previous write going on, wait for it to finish (we can't
1455 * - Submit our write and don't wait on it. We set B_WRITING indicating
1461 if (!test_bit(B_DIRTY, &b->state)) in __write_dirty_buffer()
1464 clear_bit(B_DIRTY, &b->state); in __write_dirty_buffer()
1465 wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in __write_dirty_buffer()
1467 b->write_start = b->dirty_start; in __write_dirty_buffer()
1468 b->write_end = b->dirty_end; in __write_dirty_buffer()
1473 list_add_tail(&b->write_list, write_list); in __write_dirty_buffer()
1483 list_entry(write_list->next, struct dm_buffer, write_list); in __flush_write_list()
1484 list_del(&b->write_list); in __flush_write_list()
1498 BUG_ON(atomic_read(&b->hold_count)); in __make_buffer_clean()
1501 if (!smp_load_acquire(&b->state)) /* fast case */ in __make_buffer_clean()
1504 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); in __make_buffer_clean()
1506 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in __make_buffer_clean()
1514 if (WARN_ON_ONCE(test_bit(B_WRITING, &b->state))) in is_clean()
1516 if (WARN_ON_ONCE(test_bit(B_DIRTY, &b->state))) in is_clean()
1518 if (WARN_ON_ONCE(b->list_mode != LIST_CLEAN)) in is_clean()
1521 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep && in is_clean()
1522 unlikely(test_bit(B_READING, &b->state))) in is_clean()
1531 if (WARN_ON_ONCE(test_bit(B_READING, &b->state))) in is_dirty()
1533 if (WARN_ON_ONCE(b->list_mode != LIST_DIRTY)) in is_dirty()
1547 b = cache_evict(&c->cache, LIST_CLEAN, is_clean, c); in __get_unclaimed_buffer()
1554 if (static_branch_unlikely(&no_sleep_enabled) && c->no_sleep) in __get_unclaimed_buffer()
1557 b = cache_evict(&c->cache, LIST_DIRTY, is_dirty, NULL); in __get_unclaimed_buffer()
1570 * This function is entered with c->lock held, drops it and regains it
1577 add_wait_queue(&c->free_buffer_wait, &wait); in __wait_for_free_buffer()
1583 * hold c->lock when wake_up is called. So we have a timeout here, in __wait_for_free_buffer()
1588 remove_wait_queue(&c->free_buffer_wait, &wait); in __wait_for_free_buffer()
1612 * dm-bufio is resistant to allocation failures (it just keeps in __alloc_buffer_wait_no_callback()
1621 * For debugging, if we set the cache size to 1, no new buffers will in __alloc_buffer_wait_no_callback()
1643 if (!list_empty(&c->reserved_buffers)) { in __alloc_buffer_wait_no_callback()
1644 b = list_to_buffer(c->reserved_buffers.next); in __alloc_buffer_wait_no_callback()
1645 list_del(&b->lru.list); in __alloc_buffer_wait_no_callback()
1646 c->need_reserved_buffers++; in __alloc_buffer_wait_no_callback()
1666 if (c->alloc_callback) in __alloc_buffer_wait()
1667 c->alloc_callback(b); in __alloc_buffer_wait()
1677 struct dm_bufio_client *c = b->c; in __free_buffer_wake()
1679 b->block = -1; in __free_buffer_wake()
1680 if (!c->need_reserved_buffers) in __free_buffer_wake()
1683 list_add(&b->lru.list, &c->reserved_buffers); in __free_buffer_wake()
1684 c->need_reserved_buffers--; in __free_buffer_wake()
1691 if (unlikely(waitqueue_active(&c->free_buffer_wait))) in __free_buffer_wake()
1692 wake_up(&c->free_buffer_wait); in __free_buffer_wake()
1697 if (WARN_ON_ONCE(test_bit(B_READING, &b->state))) in cleaned()
1700 if (test_bit(B_DIRTY, &b->state) || test_bit(B_WRITING, &b->state)) in cleaned()
1708 cache_mark_many(&c->cache, LIST_DIRTY, LIST_CLEAN, cleaned, NULL); in __move_clean_buffers()
1720 if (wc->no_wait && test_bit(B_WRITING, &b->state)) in write_one()
1723 __write_dirty_buffer(b, wc->write_list); in write_one()
1733 cache_iterate(&c->cache, LIST_DIRTY, write_one, &wc); in __write_dirty_buffers_async()
1739 * If we're over "limit_buffers", block until we get under the limit.
1744 if (cache_count(&c->cache, LIST_DIRTY) > in __check_watermark()
1745 cache_count(&c->cache, LIST_CLEAN) * DM_BUFIO_WRITEBACK_RATIO) in __check_watermark()
1750 *--------------------------------------------------------------
1752 *--------------------------------------------------------------
1761 if (cache_put(&c->cache, b) && in cache_put_and_wake()
1762 unlikely(waitqueue_active(&c->free_buffer_wait))) in cache_put_and_wake()
1763 wake_up(&c->free_buffer_wait); in cache_put_and_wake()
1767 * This assumes you have already checked the cache to see if the buffer
1770 static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block, in __bufio_new() argument
1790 b = cache_get(&c->cache, block); in __bufio_new()
1799 atomic_set(&b->hold_count, 1); in __bufio_new()
1800 WRITE_ONCE(b->last_accessed, jiffies); in __bufio_new()
1801 b->block = block; in __bufio_new()
1802 b->read_error = 0; in __bufio_new()
1803 b->write_error = 0; in __bufio_new()
1804 b->list_mode = LIST_CLEAN; in __bufio_new()
1807 b->state = 0; in __bufio_new()
1809 b->state = 1 << B_READING; in __bufio_new()
1814 * We mustn't insert into the cache until the B_READING state in __bufio_new()
1818 cache_insert(&c->cache, b); in __bufio_new()
1835 if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) { in __bufio_new()
1849 b->read_error = status; in read_endio()
1851 BUG_ON(!test_bit(B_READING, &b->state)); in read_endio()
1854 clear_bit(B_READING, &b->state); in read_endio()
1857 wake_up_bit(&b->state, B_READING); in read_endio()
1866 static void *new_read(struct dm_bufio_client *c, sector_t block, in new_read() argument
1878 * Fast path, hopefully the block is already in the cache. No need in new_read()
1881 b = cache_get(&c->cache, block); in new_read()
1895 if (nf == NF_GET && unlikely(test_bit_acquire(B_READING, &b->state))) { in new_read()
1906 b = __bufio_new(c, block, nf, &need_submit, &write_list); in new_read()
1911 if (b && (atomic_read(&b->hold_count) == 1)) in new_read()
1924 wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); in new_read()
1926 if (b->read_error) { in new_read()
1927 int error = blk_status_to_errno(b->read_error); in new_read()
1936 return b->data; in new_read()
1939 void *dm_bufio_get(struct dm_bufio_client *c, sector_t block, in dm_bufio_get() argument
1942 return new_read(c, block, NF_GET, bp, IOPRIO_DEFAULT); in dm_bufio_get()
1946 static void *__dm_bufio_read(struct dm_bufio_client *c, sector_t block, in __dm_bufio_read() argument
1950 return ERR_PTR(-EINVAL); in __dm_bufio_read()
1952 return new_read(c, block, NF_READ, bp, ioprio); in __dm_bufio_read()
1955 void *dm_bufio_read(struct dm_bufio_client *c, sector_t block, in dm_bufio_read() argument
1958 return __dm_bufio_read(c, block, bp, IOPRIO_DEFAULT); in dm_bufio_read()
1962 void *dm_bufio_read_with_ioprio(struct dm_bufio_client *c, sector_t block, in dm_bufio_read_with_ioprio() argument
1965 return __dm_bufio_read(c, block, bp, ioprio); in dm_bufio_read_with_ioprio()
1969 void *dm_bufio_new(struct dm_bufio_client *c, sector_t block, in dm_bufio_new() argument
1973 return ERR_PTR(-EINVAL); in dm_bufio_new()
1975 return new_read(c, block, NF_FRESH, bp, IOPRIO_DEFAULT); in dm_bufio_new()
1980 sector_t block, unsigned int n_blocks, in __dm_bufio_prefetch() argument
1992 for (; n_blocks--; block++) { in __dm_bufio_prefetch()
1996 b = cache_get(&c->cache, block); in __dm_bufio_prefetch()
1998 /* already in cache */ in __dm_bufio_prefetch()
2004 b = __bufio_new(c, block, NF_PREFETCH, &need_submit, in __dm_bufio_prefetch()
2033 void dm_bufio_prefetch(struct dm_bufio_client *c, sector_t block, unsigned int n_blocks) in dm_bufio_prefetch() argument
2035 return __dm_bufio_prefetch(c, block, n_blocks, IOPRIO_DEFAULT); in dm_bufio_prefetch()
2039 void dm_bufio_prefetch_with_ioprio(struct dm_bufio_client *c, sector_t block, in dm_bufio_prefetch_with_ioprio() argument
2042 return __dm_bufio_prefetch(c, block, n_blocks, ioprio); in dm_bufio_prefetch_with_ioprio()
2048 struct dm_bufio_client *c = b->c; in dm_bufio_release()
2055 if ((b->read_error || b->write_error) && in dm_bufio_release()
2056 !test_bit_acquire(B_READING, &b->state) && in dm_bufio_release()
2057 !test_bit(B_WRITING, &b->state) && in dm_bufio_release()
2058 !test_bit(B_DIRTY, &b->state)) { in dm_bufio_release()
2061 /* cache remove can fail if there are other holders */ in dm_bufio_release()
2062 if (cache_remove(&c->cache, b)) { in dm_bufio_release()
2078 struct dm_bufio_client *c = b->c; in dm_bufio_mark_partial_buffer_dirty()
2081 BUG_ON(end > b->c->block_size); in dm_bufio_mark_partial_buffer_dirty()
2085 BUG_ON(test_bit(B_READING, &b->state)); in dm_bufio_mark_partial_buffer_dirty()
2087 if (!test_and_set_bit(B_DIRTY, &b->state)) { in dm_bufio_mark_partial_buffer_dirty()
2088 b->dirty_start = start; in dm_bufio_mark_partial_buffer_dirty()
2089 b->dirty_end = end; in dm_bufio_mark_partial_buffer_dirty()
2090 cache_mark(&c->cache, b, LIST_DIRTY); in dm_bufio_mark_partial_buffer_dirty()
2092 if (start < b->dirty_start) in dm_bufio_mark_partial_buffer_dirty()
2093 b->dirty_start = start; in dm_bufio_mark_partial_buffer_dirty()
2094 if (end > b->dirty_end) in dm_bufio_mark_partial_buffer_dirty()
2095 b->dirty_end = end; in dm_bufio_mark_partial_buffer_dirty()
2104 dm_bufio_mark_partial_buffer_dirty(b, 0, b->c->block_size); in dm_bufio_mark_buffer_dirty()
2124 * and simultaneously (so that the block layer can merge the writes) and then
2127 * Finally, we flush hardware disk cache.
2133 return test_bit(B_WRITING, &b->state); in is_writing()
2151 nr_buffers = cache_count(&c->cache, LIST_DIRTY); in dm_bufio_write_dirty_buffers()
2152 lru_iter_begin(&c->cache.lru[LIST_DIRTY], &it); in dm_bufio_write_dirty_buffers()
2157 BUG_ON(test_bit(B_READING, &b->state)); in dm_bufio_write_dirty_buffers()
2160 nr_buffers--; in dm_bufio_write_dirty_buffers()
2162 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in dm_bufio_write_dirty_buffers()
2165 wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); in dm_bufio_write_dirty_buffers()
2168 if (!test_bit(B_DIRTY, &b->state) && !test_bit(B_WRITING, &b->state)) in dm_bufio_write_dirty_buffers()
2169 cache_mark(&c->cache, b, LIST_CLEAN); in dm_bufio_write_dirty_buffers()
2177 wake_up(&c->free_buffer_wait); in dm_bufio_write_dirty_buffers()
2180 a = xchg(&c->async_write_error, 0); in dm_bufio_write_dirty_buffers()
2190 * Use dm-io to send an empty barrier to flush the device.
2198 .client = c->dm_io, in dm_bufio_issue_flush()
2201 .bdev = c->bdev, in dm_bufio_issue_flush()
2207 return -EINVAL; in dm_bufio_issue_flush()
2214 * Use dm-io to send a discard request to flush the device.
2216 int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t count) in dm_bufio_issue_discard() argument
2222 .client = c->dm_io, in dm_bufio_issue_discard()
2225 .bdev = c->bdev, in dm_bufio_issue_discard()
2226 .sector = block_to_sector(c, block), in dm_bufio_issue_discard()
2231 return -EINVAL; /* discards are optional */ in dm_bufio_issue_discard()
2237 static bool forget_buffer(struct dm_bufio_client *c, sector_t block) in forget_buffer() argument
2241 b = cache_get(&c->cache, block); in forget_buffer()
2243 if (likely(!smp_load_acquire(&b->state))) { in forget_buffer()
2244 if (cache_remove(&c->cache, b)) in forget_buffer()
2262 void dm_bufio_forget(struct dm_bufio_client *c, sector_t block) in dm_bufio_forget() argument
2265 forget_buffer(c, block); in dm_bufio_forget()
2272 return b->state ? ER_DONT_EVICT : ER_EVICT; in idle()
2275 void dm_bufio_forget_buffers(struct dm_bufio_client *c, sector_t block, sector_t n_blocks) in dm_bufio_forget_buffers() argument
2278 cache_remove_range(&c->cache, block, block + n_blocks, idle, __free_buffer_wake); in dm_bufio_forget_buffers()
2285 c->minimum_buffers = n; in dm_bufio_set_minimum_buffers()
2291 return c->block_size; in dm_bufio_get_block_size()
2297 sector_t s = bdev_nr_sectors(c->bdev); in dm_bufio_get_device_size()
2299 if (s >= c->start) in dm_bufio_get_device_size()
2300 s -= c->start; in dm_bufio_get_device_size()
2303 if (likely(c->sectors_per_block_bits >= 0)) in dm_bufio_get_device_size()
2304 s >>= c->sectors_per_block_bits; in dm_bufio_get_device_size()
2306 sector_div(s, c->block_size >> SECTOR_SHIFT); in dm_bufio_get_device_size()
2313 return c->dm_io; in dm_bufio_get_dm_io_client()
2319 return b->block; in dm_bufio_get_block_number()
2325 return b->data; in dm_bufio_get_block_data()
2337 return b->c; in dm_bufio_get_client()
2348 (unsigned long long)b->block, atomic_read(&b->hold_count), b->list_mode); in warn_leak()
2350 stack_trace_print(b->stack_entries, b->stack_len, 1); in warn_leak()
2352 atomic_set(&b->hold_count, 0); in warn_leak()
2366 * An optimization so that the buffers are not written one-by-one. in drop_buffers()
2378 cache_iterate(&c->cache, i, warn_leak, &warned); in drop_buffers()
2387 WARN_ON(cache_count(&c->cache, i)); in drop_buffers()
2396 if (likely(c->sectors_per_block_bits >= 0)) in get_retain_buffers()
2397 retain_bytes >>= c->sectors_per_block_bits + SECTOR_SHIFT; in get_retain_buffers()
2399 retain_bytes /= c->block_size; in get_retain_buffers()
2410 unsigned long count = cache_total(&c->cache); in __scan()
2414 if (count - freed <= retain_target) in __scan()
2415 atomic_long_set(&c->need_shrink, 0); in __scan()
2416 if (!atomic_long_read(&c->need_shrink)) in __scan()
2419 b = cache_evict(&c->cache, l, in __scan()
2427 atomic_long_dec(&c->need_shrink); in __scan()
2447 c = shrink->private_data; in dm_bufio_shrink_scan()
2448 atomic_long_add(sc->nr_to_scan, &c->need_shrink); in dm_bufio_shrink_scan()
2449 queue_work(dm_bufio_wq, &c->shrink_work); in dm_bufio_shrink_scan()
2451 return sc->nr_to_scan; in dm_bufio_shrink_scan()
2456 struct dm_bufio_client *c = shrink->private_data; in dm_bufio_shrink_count()
2457 unsigned long count = cache_total(&c->cache); in dm_bufio_shrink_count()
2459 unsigned long queued_for_cleanup = atomic_long_read(&c->need_shrink); in dm_bufio_shrink_count()
2464 count -= retain_target; in dm_bufio_shrink_count()
2469 count -= queued_for_cleanup; in dm_bufio_shrink_count()
2489 if (!block_size || block_size & ((1 << SECTOR_SHIFT) - 1)) { in dm_bufio_client_create()
2490 DMERR("%s: block size not specified or is not multiple of 512b", __func__); in dm_bufio_client_create()
2491 r = -EINVAL; in dm_bufio_client_create()
2498 r = -ENOMEM; in dm_bufio_client_create()
2501 cache_init(&c->cache, num_locks, (flags & DM_BUFIO_CLIENT_NO_SLEEP) != 0); in dm_bufio_client_create()
2503 c->bdev = bdev; in dm_bufio_client_create()
2504 c->block_size = block_size; in dm_bufio_client_create()
2506 c->sectors_per_block_bits = __ffs(block_size) - SECTOR_SHIFT; in dm_bufio_client_create()
2508 c->sectors_per_block_bits = -1; in dm_bufio_client_create()
2510 c->alloc_callback = alloc_callback; in dm_bufio_client_create()
2511 c->write_callback = write_callback; in dm_bufio_client_create()
2514 c->no_sleep = true; in dm_bufio_client_create()
2518 mutex_init(&c->lock); in dm_bufio_client_create()
2519 spin_lock_init(&c->spinlock); in dm_bufio_client_create()
2520 INIT_LIST_HEAD(&c->reserved_buffers); in dm_bufio_client_create()
2521 c->need_reserved_buffers = reserved_buffers; in dm_bufio_client_create()
2525 init_waitqueue_head(&c->free_buffer_wait); in dm_bufio_client_create()
2526 c->async_write_error = 0; in dm_bufio_client_create()
2528 c->dm_io = dm_io_client_create(); in dm_bufio_client_create()
2529 if (IS_ERR(c->dm_io)) { in dm_bufio_client_create()
2530 r = PTR_ERR(c->dm_io); in dm_bufio_client_create()
2537 snprintf(slab_name, sizeof(slab_name), "dm_bufio_cache-%u-%u", in dm_bufio_client_create()
2539 c->slab_cache = kmem_cache_create(slab_name, block_size, align, in dm_bufio_client_create()
2541 if (!c->slab_cache) { in dm_bufio_client_create()
2542 r = -ENOMEM; in dm_bufio_client_create()
2547 snprintf(slab_name, sizeof(slab_name), "dm_bufio_buffer-%u-%u", in dm_bufio_client_create()
2550 snprintf(slab_name, sizeof(slab_name), "dm_bufio_buffer-%u", in dm_bufio_client_create()
2552 c->slab_buffer = kmem_cache_create(slab_name, sizeof(struct dm_buffer) + aux_size, in dm_bufio_client_create()
2554 if (!c->slab_buffer) { in dm_bufio_client_create()
2555 r = -ENOMEM; in dm_bufio_client_create()
2559 while (c->need_reserved_buffers) { in dm_bufio_client_create()
2563 r = -ENOMEM; in dm_bufio_client_create()
2569 INIT_WORK(&c->shrink_work, shrink_work); in dm_bufio_client_create()
2570 atomic_long_set(&c->need_shrink, 0); in dm_bufio_client_create()
2572 c->shrinker = shrinker_alloc(0, "dm-bufio:(%u:%u)", in dm_bufio_client_create()
2573 MAJOR(bdev->bd_dev), MINOR(bdev->bd_dev)); in dm_bufio_client_create()
2574 if (!c->shrinker) { in dm_bufio_client_create()
2575 r = -ENOMEM; in dm_bufio_client_create()
2579 c->shrinker->count_objects = dm_bufio_shrink_count; in dm_bufio_client_create()
2580 c->shrinker->scan_objects = dm_bufio_shrink_scan; in dm_bufio_client_create()
2581 c->shrinker->seeks = 1; in dm_bufio_client_create()
2582 c->shrinker->batch = 0; in dm_bufio_client_create()
2583 c->shrinker->private_data = c; in dm_bufio_client_create()
2585 shrinker_register(c->shrinker); in dm_bufio_client_create()
2589 list_add(&c->client_list, &dm_bufio_all_clients); in dm_bufio_client_create()
2596 while (!list_empty(&c->reserved_buffers)) { in dm_bufio_client_create()
2597 struct dm_buffer *b = list_to_buffer(c->reserved_buffers.next); in dm_bufio_client_create()
2599 list_del(&b->lru.list); in dm_bufio_client_create()
2602 kmem_cache_destroy(c->slab_cache); in dm_bufio_client_create()
2603 kmem_cache_destroy(c->slab_buffer); in dm_bufio_client_create()
2604 dm_io_client_destroy(c->dm_io); in dm_bufio_client_create()
2606 mutex_destroy(&c->lock); in dm_bufio_client_create()
2607 if (c->no_sleep) in dm_bufio_client_create()
2625 shrinker_free(c->shrinker); in dm_bufio_client_destroy()
2626 flush_work(&c->shrink_work); in dm_bufio_client_destroy()
2630 list_del(&c->client_list); in dm_bufio_client_destroy()
2631 dm_bufio_client_count--; in dm_bufio_client_destroy()
2636 WARN_ON(c->need_reserved_buffers); in dm_bufio_client_destroy()
2638 while (!list_empty(&c->reserved_buffers)) { in dm_bufio_client_destroy()
2639 struct dm_buffer *b = list_to_buffer(c->reserved_buffers.next); in dm_bufio_client_destroy()
2641 list_del(&b->lru.list); in dm_bufio_client_destroy()
2646 if (cache_count(&c->cache, i)) in dm_bufio_client_destroy()
2647 DMERR("leaked buffer count %d: %lu", i, cache_count(&c->cache, i)); in dm_bufio_client_destroy()
2650 WARN_ON(cache_count(&c->cache, i)); in dm_bufio_client_destroy()
2652 cache_destroy(&c->cache); in dm_bufio_client_destroy()
2653 kmem_cache_destroy(c->slab_cache); in dm_bufio_client_destroy()
2654 kmem_cache_destroy(c->slab_buffer); in dm_bufio_client_destroy()
2655 dm_io_client_destroy(c->dm_io); in dm_bufio_client_destroy()
2656 mutex_destroy(&c->lock); in dm_bufio_client_destroy()
2657 if (c->no_sleep) in dm_bufio_client_destroy()
2666 flush_work(&c->shrink_work); in dm_bufio_client_reset()
2672 c->start = start; in dm_bufio_set_sector_offset()
2676 /*--------------------------------------------------------------*/
2690 return time_after_eq(jiffies, READ_ONCE(b->last_accessed) + age_hz); in older_than()
2717 if (!(params->gfp & __GFP_FS) || in select_for_evict()
2718 (static_branch_unlikely(&no_sleep_enabled) && b->c->no_sleep)) { in select_for_evict()
2719 if (test_bit_acquire(B_READING, &b->state) || in select_for_evict()
2720 test_bit(B_WRITING, &b->state) || in select_for_evict()
2721 test_bit(B_DIRTY, &b->state)) in select_for_evict()
2725 return older_than(b, params->age_hz) ? ER_EVICT : ER_STOP; in select_for_evict()
2737 b = cache_evict(&c->cache, list_mode, select_for_evict, params); in __evict_many()
2741 last_accessed = READ_ONCE(b->last_accessed); in __evict_many()
2742 if (time_after_eq(params->last_accessed, last_accessed)) in __evict_many()
2743 params->last_accessed = last_accessed; in __evict_many()
2770 count = cache_total(&c->cache); in evict_old_buffers()
2772 __evict_many(c, ¶ms, LIST_CLEAN, count - retain); in evict_old_buffers()
2800 /*--------------------------------------------------------------*/
2831 if (time_after_eq(c->oldest_buffer, new_client->oldest_buffer)) in __insert_client()
2833 h = h->next; in __insert_client()
2836 list_add_tail(&new_client->client_list, h); in __insert_client()
2859 c->oldest_buffer = params.last_accessed; in __evict_a_few()
2883 unsigned long threshold = dm_bufio_cache_size - in evict_old()
2902 *--------------------------------------------------------------
2904 *--------------------------------------------------------------
2921 mem = (__u64)mult_frac(totalram_pages() - totalhigh_pages(), in dm_bufio_init()
2940 return -ENOMEM; in dm_bufio_init()
2991 MODULE_PARM_DESC(max_cache_size_bytes, "Size of metadata cache");
3015 MODULE_PARM_DESC(current_allocated_bytes, "Memory currently used by the cache");
3017 MODULE_AUTHOR("Mikulas Patocka <dm-[email protected]>");