Lines Matching full:unlikely
470 if (unlikely(*sec_ptr >= ic->journal_sections)) in wraparound_section()
508 if (unlikely(r < 0)) { in sb_mac()
516 if (unlikely(r < 0)) { in sb_mac()
550 if (unlikely(r)) in sync_rw_sb()
556 if (unlikely(r)) in sync_rw_sb()
562 if (unlikely(r)) in sync_rw_sb()
581 if (unlikely(((sector | n_sectors) & ((1 << ic->sb->log2_sectors_per_block) - 1)) != 0)) { in block_bitmap_op()
591 if (unlikely(!n_sectors)) in block_bitmap_op()
672 if (unlikely(page < end_page)) { in block_bitmap_op()
709 if (unlikely(section >= ic->journal_sections) || in access_journal_check()
710 unlikely(offset >= limit)) { in access_journal_check()
786 if (unlikely(r < 0)) { in section_mac()
795 if (unlikely(r < 0)) { in section_mac()
802 if (unlikely(r < 0)) { in section_mac()
812 if (unlikely(r < 0)) { in section_mac()
822 if (unlikely(r < 0)) { in section_mac()
835 if (unlikely(r < 0)) { in section_mac()
909 while (unlikely(pl_index == section_index)) { in xor_journal()
942 if (unlikely(err)) { in complete_journal_encrypt()
1008 if (unlikely(do_crypt(encrypt, req, comp))) in crypt_journal()
1032 if (unlikely(error != 0)) in complete_journal_io()
1046 if (unlikely(dm_integrity_failed(ic))) { in rw_journal_sectors()
1074 if (unlikely(r)) { in rw_journal_sectors()
1169 if (unlikely(dm_integrity_failed(ic))) { in copy_from_journal()
1191 if (unlikely(r)) { in copy_from_journal()
1214 if (unlikely(ranges_overlap(range, new_range))) in add_new_range()
1242 while (unlikely(!list_empty(&ic->wait_list))) { in remove_range_unlocked()
1278 } while (unlikely(new_range->waiting)); in wait_and_add_new_range()
1283 if (unlikely(!add_new_range(ic, new_range, true))) in add_new_range_and_wait()
1355 if (unlikely(pos >= ic->journal_entries)) in test_journal_node()
1358 if (unlikely(RB_EMPTY_NODE(&node->node))) in test_journal_node()
1360 if (unlikely(node->sector != sector)) in test_journal_node()
1364 if (unlikely(!next)) in test_journal_node()
1380 if (unlikely(!next)) in find_newer_committed_node()
1416 if (unlikely(r)) in dm_integrity_rw_tag()
1436 if (unlikely(crypto_memneq(dp, tag, to_copy))) in dm_integrity_rw_tag()
1452 if (unlikely(hash_offset == ic->tag_size)) { in dm_integrity_rw_tag()
1453 if (unlikely(mismatch_hash) && unlikely(mismatch_filler)) { in dm_integrity_rw_tag()
1468 if (unlikely(*metadata_offset == 1U << SECTOR_SHIFT << ic->log2_buffer_sectors)) { in dm_integrity_rw_tag()
1473 if (unlikely(!is_power_of_2(ic->tag_size))) in dm_integrity_rw_tag()
1477 } while (unlikely(total_size)); in dm_integrity_rw_tag()
1493 if (unlikely(error != 0)) in flush_notify()
1522 if (unlikely(r)) in dm_integrity_flush_buffers()
1573 if (unlikely(r) && !bio->bi_status) in do_endio()
1575 if (unlikely(ic->synchronous_mode) && bio_op(bio) == REQ_OP_WRITE) { in do_endio()
1591 if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic))) in do_endio_flush()
1605 if (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD)) in dec_in_flight()
1609 if (unlikely(dio->bi_status) && !bio->bi_status) in dec_in_flight()
1611 if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) { in dec_in_flight()
1647 if (unlikely(r < 0)) { in integrity_sector_checksum()
1654 if (unlikely(r < 0)) { in integrity_sector_checksum()
1661 if (unlikely(r < 0)) { in integrity_sector_checksum()
1667 if (unlikely(r < 0)) { in integrity_sector_checksum()
1673 if (unlikely(r < 0)) { in integrity_sector_checksum()
1679 if (unlikely(digest_size < ic->tag_size)) in integrity_sector_checksum()
1734 if (unlikely(r)) { in integrity_recheck()
1781 unsigned int extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0; in integrity_metadata()
1786 if (unlikely(ic->mode == 'R')) in integrity_metadata()
1803 if (unlikely(dio->op == REQ_OP_DISCARD)) { in integrity_metadata()
1816 if (unlikely(r)) { in integrity_metadata()
1853 if (unlikely(r)) { in integrity_metadata()
1866 if (unlikely(pos < bv_copy.bv_len)) { in integrity_metadata()
1895 if (unlikely(r)) in integrity_metadata()
1913 if (unlikely(logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) { in dm_integrity_check_limits()
1919 if (unlikely((logical_sector | bio_sectors(bio)) & (unsigned int)(ic->sectors_per_block - 1))) { in dm_integrity_check_limits()
1930 if (unlikely(bv.bv_len & ((ic->sectors_per_block << SECTOR_SHIFT) - 1))) { in dm_integrity_check_limits()
1960 if (unlikely(dio->op == REQ_OP_DISCARD)) { in dm_integrity_map()
1975 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) { in dm_integrity_map()
1982 if (unlikely(dio->fua)) { in dm_integrity_map()
1989 if (unlikely(!dm_integrity_check_limits(ic, dio->range.logical_sector, bio))) in dm_integrity_map()
2001 if (unlikely(wanted_tag_size != bip->bip_iter.bi_size)) { in dm_integrity_map()
2008 if (unlikely(bip != NULL)) { in dm_integrity_map()
2014 if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ)) in dm_integrity_map()
2038 if (unlikely(bv.bv_len >> SECTOR_SHIFT > n_sectors)) in __journal_read_write()
2050 if (unlikely(dio->op == REQ_OP_READ)) { in __journal_read_write()
2055 if (unlikely(journal_entry_is_inprogress(je))) { in __journal_read_write()
2078 if (unlikely(crypto_memneq(checksums_onstack, journal_entry_tag(ic, je), ic->tag_size))) { in __journal_read_write()
2108 } while (unlikely(tag_todo)); in __journal_read_write()
2128 if (unlikely(digest_size > ic->tag_size)) { in __journal_read_write()
2142 if (unlikely(journal_entry == ic->journal_section_entries)) { in __journal_read_write()
2151 if (unlikely(dio->op == REQ_OP_READ)) in __journal_read_write()
2158 if (unlikely(waitqueue_active(&ic->copy_to_journal_wait))) in __journal_read_write()
2167 if (unlikely(bio->bi_iter.bi_size)) { in __journal_read_write()
2190 if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D') in dm_integrity_map_continue()
2202 if (unlikely(dm_integrity_failed(ic))) { in dm_integrity_map_continue()
2216 if (unlikely(!dio->range.n_sectors)) { in dm_integrity_map_continue()
2242 if (unlikely(pos >= ic->journal_entries)) in dm_integrity_map_continue()
2249 if (unlikely(we == ic->journal_section_entries)) { in dm_integrity_map_continue()
2263 if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector)) in dm_integrity_map_continue()
2277 if (unlikely(!add_new_range(ic, &dio->range, true))) { in dm_integrity_map_continue()
2303 if (unlikely(new_pos != journal_read_pos)) { in dm_integrity_map_continue()
2314 if (unlikely(new_pos != NOT_FOUND) || in dm_integrity_map_continue()
2315 unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) { in dm_integrity_map_continue()
2329 if (unlikely(journal_read_pos != NOT_FOUND)) { in dm_integrity_map_continue()
2335 if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) { in dm_integrity_map_continue()
2364 if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) { in dm_integrity_map_continue()
2402 if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry))) in dm_integrity_map_continue()
2416 if (unlikely(bio_integrity(bio))) { in dm_integrity_map_inline()
2423 if (unlikely((bio->bi_opf & REQ_PREFLUSH) != 0)) in dm_integrity_map_inline()
2431 extra_size = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0; in dm_integrity_map_inline()
2434 if (unlikely(!dio->integrity_payload)) { in dm_integrity_map_inline()
2469 if (unlikely(!add_new_range(ic, &dio->range, true))) { in dm_integrity_map_inline()
2483 if (unlikely(!dio->integrity_payload)) { in dm_integrity_map_inline()
2490 if (unlikely(!dm_integrity_check_limits(ic, bio->bi_iter.bi_sector, bio))) { in dm_integrity_map_inline()
2519 if (unlikely(ret != dio->payload_len)) { in dm_integrity_map_inline()
2531 if (unlikely(dio->integrity_payload_from_mempool)) in dm_integrity_free_payload()
2562 if (unlikely(r != (ic->sectors_per_block << SECTOR_SHIFT))) { in dm_integrity_inline_recheck()
2578 if (unlikely(r != ic->tuple_size)) { in dm_integrity_inline_recheck()
2588 if (unlikely(r != 0)) { in dm_integrity_inline_recheck()
2597 …if (unlikely(crypto_memneq(digest, dio->integrity_payload, min(crypto_shash_digestsize(ic->interna… in dm_integrity_inline_recheck()
2628 unlikely(dio->integrity_range_locked)) in dm_integrity_end_io()
2636 if (unlikely(crypto_memneq(digest, dio->integrity_payload + pos, in dm_integrity_end_io()
2651 if (unlikely(dio->integrity_range_locked)) in dm_integrity_end_io()
2717 if (unlikely(ic->mode != 'J')) { in integrity_commit()
2748 if (unlikely(i >= ic->journal_sections)) in integrity_commit()
2784 if (unlikely(error != 0)) in complete_copy_from_journal()
2816 if (unlikely(from_replay)) in do_journal_write()
2829 BUG_ON(unlikely(journal_entry_is_inprogress(je)) && !from_replay); in do_journal_write()
2831 if (unlikely(from_replay)) { in do_journal_write()
2832 if (unlikely(sec & (unsigned int)(ic->sectors_per_block - 1))) { in do_journal_write()
2836 if (unlikely(sec >= ic->provided_data_sectors)) { in do_journal_write()
2849 BUG_ON(unlikely(journal_entry_is_inprogress(je2)) && !from_replay); in do_journal_write()
2851 if (unlikely(sec2 >= ic->provided_data_sectors)) in do_journal_write()
2906 unlikely(from_replay) && in do_journal_write()
2913 if (unlikely(crypto_memneq(test_tag, journal_entry_tag(ic, je2), ic->tag_size))) { in do_journal_write()
2922 if (unlikely(r)) in do_journal_write()
2970 if (unlikely(!prev_free_sectors)) in integrity_writer()
2985 if (unlikely(r)) in recalc_write_super()
3034 if (unlikely(dm_post_suspending(ic->ti))) in integrity_recalc()
3038 if (unlikely(range.logical_sector >= ic->provided_data_sectors)) { in integrity_recalc()
3077 if (unlikely(++super_counter == RECALC_WRITE_SUPER)) { in integrity_recalc()
3085 if (unlikely(dm_integrity_failed(ic))) in integrity_recalc()
3098 if (unlikely(r)) { in integrity_recalc()
3112 if (unlikely(r)) { in integrity_recalc()
3191 if (unlikely(dm_post_suspending(ic->ti))) in integrity_recalc_inline()
3195 if (unlikely(range.logical_sector >= ic->provided_data_sectors)) in integrity_recalc_inline()
3202 if (unlikely(++super_counter == RECALC_WRITE_SUPER)) { in integrity_recalc_inline()
3207 if (unlikely(dm_integrity_failed(ic))) in integrity_recalc_inline()
3217 if (unlikely(r)) { in integrity_recalc_inline()
3234 if (unlikely(IS_ERR(bip))) { in integrity_recalc_inline()
3240 if (unlikely(ret != t - recalc_tags)) { in integrity_recalc_inline()
3248 if (unlikely(r)) { in integrity_recalc_inline()
3364 while (unlikely((bio = bio_list_pop(&ic->synchronous_bios)) != NULL)) { in bitmap_flush_work()
3502 if (unlikely(write_start >= ic->journal_sections)) in replay_journal()
3523 if (unlikely(i >= ic->journal_sections)) in replay_journal()
3641 if (unlikely(r)) in dm_integrity_postsuspend()
3675 if (unlikely(r)) in dm_integrity_resume()
3717 if (unlikely(r)) in dm_integrity_resume()
3729 if (unlikely(r)) in dm_integrity_resume()
5124 if (unlikely(r)) { in dm_integrity_ctr()