1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * Copyright (C) 2022-2023 Oracle.  All Rights Reserved.
130 	error = xfs_rmap_alloc(sc->tp, sc->sa.agf_bp, sc->sa.pag, agbno, 1,  in xreap_put_freelist()
136 	error = xfs_alloc_read_agfl(sc->sa.pag, sc->tp, &agfl_bp);  in xreap_put_freelist()
140 	error = xfs_alloc_put_freelist(sc->sa.pag, sc->tp, sc->sa.agf_bp,  in xreap_put_freelist()
144 	xfs_extent_busy_insert(sc->tp, pag_group(sc->sa.pag), agbno, 1,  in xreap_put_freelist()
151 static inline bool xreap_dirty(const struct xreap_state *rs)  in xreap_dirty()  argument
153 	if (rs->force_roll)  in xreap_dirty()
155 	if (rs->deferred)  in xreap_dirty()
157 	if (rs->invalidated)  in xreap_dirty()
159 	if (rs->total_deferred)  in xreap_dirty()
172 static inline bool xreap_want_roll(const struct xreap_state *rs)  in xreap_want_roll()  argument
174 	if (rs->force_roll)  in xreap_want_roll()
176 	if (rs->deferred > XREP_MAX_ITRUNCATE_EFIS)  in xreap_want_roll()
178 	if (rs->invalidated > XREAP_MAX_BINVAL)  in xreap_want_roll()
183 static inline void xreap_reset(struct xreap_state *rs)  in xreap_reset()  argument
185 	rs->total_deferred += rs->deferred;  in xreap_reset()
186 	rs->deferred = 0;  in xreap_reset()
187 	rs->invalidated = 0;  in xreap_reset()
188 	rs->force_roll = false;  in xreap_reset()
201 xreap_want_defer_finish(const struct xreap_state *rs)  in xreap_want_defer_finish()  argument
203 	if (rs->force_roll)  in xreap_want_defer_finish()
205 	if (rs->total_deferred > XREAP_MAX_DEFER_CHAIN)  in xreap_want_defer_finish()
210 static inline void xreap_defer_finish_reset(struct xreap_state *rs)  in xreap_defer_finish_reset()  argument
212 	rs->total_deferred = 0;  in xreap_defer_finish_reset()
213 	rs->deferred = 0;  in xreap_defer_finish_reset()
214 	rs->invalidated = 0;  in xreap_defer_finish_reset()
215 	rs->force_roll = false;  in xreap_defer_finish_reset()
244 	scan->__sector_count += scan->daddr_step;  in xrep_bufscan_advance()
245 	while (scan->__sector_count <= scan->max_sectors) {  in xrep_bufscan_advance()
249 		error = xfs_buf_incore(mp->m_ddev_targp, scan->daddr,  in xrep_bufscan_advance()
250 				scan->__sector_count, XBF_LIVESCAN, &bp);  in xrep_bufscan_advance()
254 		scan->__sector_count += scan->daddr_step;  in xrep_bufscan_advance()
263 	struct xreap_state	*rs,  in xreap_agextent_binval()  argument
267 	struct xfs_scrub	*sc = rs->sc;  in xreap_agextent_binval()
268 	struct xfs_perag	*pag = sc->sa.pag;  in xreap_agextent_binval()
269 	struct xfs_mount	*mp = sc->mp;  in xreap_agextent_binval()
274 	 * Avoid invalidating AG headers and post-EOFS blocks because we never  in xreap_agextent_binval()
278 	    !xfs_verify_agbno(pag, agbno_next - 1))  in xreap_agextent_binval()
292 							agbno_next - bno),  in xreap_agextent_binval()
298 			xfs_trans_bjoin(sc->tp, bp);  in xreap_agextent_binval()
299 			xfs_trans_binval(sc->tp, bp);  in xreap_agextent_binval()
300 			rs->invalidated++;  in xreap_agextent_binval()
307 			if (rs->invalidated > XREAP_MAX_BINVAL) {  in xreap_agextent_binval()
308 				*aglenp -= agbno_next - bno;  in xreap_agextent_binval()
317 	trace_xreap_agextent_binval(pag_group(sc->sa.pag), agbno, *aglenp);  in xreap_agextent_binval()
322  * call.  Cross-linked blocks should have their reverse mappings removed, but
323  * single-owner extents can be freed.  AGFL blocks can only be put back one at
328 	struct xreap_state	*rs,  in xreap_agextent_select()  argument
334 	struct xfs_scrub	*sc = rs->sc;  in xreap_agextent_select()
344 	cur = xfs_rmapbt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp,  in xreap_agextent_select()
345 			sc->sa.pag);  in xreap_agextent_select()
346 	error = xfs_rmap_has_other_keys(cur, agbno, 1, rs->oinfo,  in xreap_agextent_select()
352 	if (rs->resv == XFS_AG_RESV_AGFL)  in xreap_agextent_select()
362 		error = xfs_rmap_has_other_keys(cur, bno, 1, rs->oinfo,  in xreap_agextent_select()
376 	trace_xreap_agextent_select(pag_group(sc->sa.pag), agbno, len,  in xreap_agextent_select()
389 	struct xreap_state	*rs,  in xreap_agextent_iter()  argument
394 	struct xfs_scrub	*sc = rs->sc;  in xreap_agextent_iter()
398 	ASSERT(rs->resv != XFS_AG_RESV_METAFILE);  in xreap_agextent_iter()
400 	fsbno = xfs_agbno_to_fsb(sc->sa.pag, agbno);  in xreap_agextent_iter()
409 	 * metadata structure is crosslinked with a multi-block structure  in xreap_agextent_iter()
416 		trace_xreap_dispose_unmap_extent(pag_group(sc->sa.pag), agbno,  in xreap_agextent_iter()
419 		rs->force_roll = true;  in xreap_agextent_iter()
421 		if (rs->oinfo == &XFS_RMAP_OINFO_COW) {  in xreap_agextent_iter()
427 			xfs_refcount_free_cow_extent(sc->tp, false, fsbno,  in xreap_agextent_iter()
432 		return xfs_rmap_free(sc->tp, sc->sa.agf_bp, sc->sa.pag, agbno,  in xreap_agextent_iter()
433 				*aglenp, rs->oinfo);  in xreap_agextent_iter()
436 	trace_xreap_dispose_free_extent(pag_group(sc->sa.pag), agbno, *aglenp);  in xreap_agextent_iter()
444 	xreap_agextent_binval(rs, agbno, aglenp);  in xreap_agextent_iter()
446 		ASSERT(xreap_want_roll(rs));  in xreap_agextent_iter()
457 	if (rs->oinfo == &XFS_RMAP_OINFO_COW) {  in xreap_agextent_iter()
458 		ASSERT(rs->resv == XFS_AG_RESV_NONE);  in xreap_agextent_iter()
460 		xfs_refcount_free_cow_extent(sc->tp, false, fsbno, *aglenp);  in xreap_agextent_iter()
461 		error = xfs_free_extent_later(sc->tp, fsbno, *aglenp, NULL,  in xreap_agextent_iter()
462 				rs->resv, XFS_FREE_EXTENT_SKIP_DISCARD);  in xreap_agextent_iter()
466 		rs->force_roll = true;  in xreap_agextent_iter()
471 	if (rs->resv == XFS_AG_RESV_AGFL) {  in xreap_agextent_iter()
477 		rs->force_roll = true;  in xreap_agextent_iter()
487 	error = xfs_free_extent_later(sc->tp, fsbno, *aglenp, rs->oinfo,  in xreap_agextent_iter()
488 			rs->resv, XFS_FREE_EXTENT_SKIP_DISCARD);  in xreap_agextent_iter()
492 	rs->deferred++;  in xreap_agextent_iter()
493 	if (rs->deferred % 2 == 0)  in xreap_agextent_iter()
494 		xfs_defer_add_barrier(sc->tp);  in xreap_agextent_iter()
499  * Break an AG metadata extent into sub-extents by fate (crosslinked, not
500  * crosslinked), and dispose of each sub-extent separately.
508 	struct xreap_state	*rs = priv;  in xreap_agmeta_extent()  local
509 	struct xfs_scrub	*sc = rs->sc;  in xreap_agmeta_extent()
514 	ASSERT(sc->ip == NULL);  in xreap_agmeta_extent()
520 		error = xreap_agextent_select(rs, agbno, agbno_next,  in xreap_agmeta_extent()
525 		error = xreap_agextent_iter(rs, agbno, &aglen, crosslinked);  in xreap_agmeta_extent()
529 		if (xreap_want_defer_finish(rs)) {  in xreap_agmeta_extent()
533 			xreap_defer_finish_reset(rs);  in xreap_agmeta_extent()
534 		} else if (xreap_want_roll(rs)) {  in xreap_agmeta_extent()
538 			xreap_reset(rs);  in xreap_agmeta_extent()
555 	struct xreap_state		rs = {  in xrep_reap_agblocks()  local
562 	ASSERT(xfs_has_rmapbt(sc->mp));  in xrep_reap_agblocks()
563 	ASSERT(sc->ip == NULL);  in xrep_reap_agblocks()
565 	error = xagb_bitmap_walk(bitmap, xreap_agmeta_extent, &rs);  in xrep_reap_agblocks()
569 	if (xreap_dirty(&rs))  in xrep_reap_agblocks()
576  * Break a file metadata extent into sub-extents by fate (crosslinked, not
577  * crosslinked), and dispose of each sub-extent separately.  The extent must
586 	struct xreap_state	*rs = priv;  in xreap_fsmeta_extent()  local
587 	struct xfs_scrub	*sc = rs->sc;  in xreap_fsmeta_extent()
588 	xfs_agnumber_t		agno = XFS_FSB_TO_AGNO(sc->mp, fsbno);  in xreap_fsmeta_extent()
589 	xfs_agblock_t		agbno = XFS_FSB_TO_AGBNO(sc->mp, fsbno);  in xreap_fsmeta_extent()
594 	ASSERT(sc->ip != NULL);  in xreap_fsmeta_extent()
595 	ASSERT(!sc->sa.pag);  in xreap_fsmeta_extent()
601 	sc->sa.pag = xfs_perag_get(sc->mp, agno);  in xreap_fsmeta_extent()
602 	if (!sc->sa.pag)  in xreap_fsmeta_extent()
603 		return -EFSCORRUPTED;  in xreap_fsmeta_extent()
605 	error = xfs_alloc_read_agf(sc->sa.pag, sc->tp, 0, &sc->sa.agf_bp);  in xreap_fsmeta_extent()
613 		error = xreap_agextent_select(rs, agbno, agbno_next,  in xreap_fsmeta_extent()
618 		error = xreap_agextent_iter(rs, agbno, &aglen, crosslinked);  in xreap_fsmeta_extent()
622 		if (xreap_want_defer_finish(rs)) {  in xreap_fsmeta_extent()
630 			xreap_defer_finish_reset(rs);  in xreap_fsmeta_extent()
631 		} else if (xreap_want_roll(rs)) {  in xreap_fsmeta_extent()
637 			xfs_trans_bhold(sc->tp, sc->sa.agf_bp);  in xreap_fsmeta_extent()
638 			error = xfs_trans_roll_inode(&sc->tp, sc->ip);  in xreap_fsmeta_extent()
639 			xfs_trans_bjoin(sc->tp, sc->sa.agf_bp);  in xreap_fsmeta_extent()
642 			xreap_reset(rs);  in xreap_fsmeta_extent()
649 	xfs_trans_brelse(sc->tp, sc->sa.agf_bp);  in xreap_fsmeta_extent()
650 	sc->sa.agf_bp = NULL;  in xreap_fsmeta_extent()
652 	xfs_perag_put(sc->sa.pag);  in xreap_fsmeta_extent()
653 	sc->sa.pag = NULL;  in xreap_fsmeta_extent()
667 	struct xreap_state		rs = {  in xrep_reap_fsblocks()  local
674 	ASSERT(xfs_has_rmapbt(sc->mp));  in xrep_reap_fsblocks()
675 	ASSERT(sc->ip != NULL);  in xrep_reap_fsblocks()
677 	error = xfsb_bitmap_walk(bitmap, xreap_fsmeta_extent, &rs);  in xrep_reap_fsblocks()
681 	if (xreap_dirty(&rs))  in xrep_reap_fsblocks()
690  * call.  Cross-linked blocks should have their reverse mappings removed, but
691  * single-owner extents can be freed.  Units are rt blocks, not rt extents.
695 	struct xreap_state	*rs,  in xreap_rgextent_select()  argument
701 	struct xfs_scrub	*sc = rs->sc;  in xreap_rgextent_select()
711 	cur = xfs_rtrmapbt_init_cursor(sc->tp, sc->sr.rtg);  in xreap_rgextent_select()
712 	error = xfs_rmap_has_other_keys(cur, rgbno, 1, rs->oinfo,  in xreap_rgextent_select()
724 		error = xfs_rmap_has_other_keys(cur, bno, 1, rs->oinfo,  in xreap_rgextent_select()
737 	trace_xreap_agextent_select(rtg_group(sc->sr.rtg), rgbno, len,  in xreap_rgextent_select()
750 	struct xreap_state	*rs,  in xreap_rgextent_iter()  argument
755 	struct xfs_scrub	*sc = rs->sc;  in xreap_rgextent_iter()
764 	if (rs->oinfo != &XFS_RMAP_OINFO_COW) {  in xreap_rgextent_iter()
765 		ASSERT(rs->oinfo == &XFS_RMAP_OINFO_COW);  in xreap_rgextent_iter()
766 		return -EFSCORRUPTED;  in xreap_rgextent_iter()
768 	ASSERT(rs->resv == XFS_AG_RESV_NONE);  in xreap_rgextent_iter()
770 	rtbno = xfs_rgbno_to_rtb(sc->sr.rtg, rgbno);  in xreap_rgextent_iter()
777 		trace_xreap_dispose_unmap_extent(rtg_group(sc->sr.rtg), rgbno,  in xreap_rgextent_iter()
780 		xfs_refcount_free_cow_extent(sc->tp, true, rtbno, *rglenp);  in xreap_rgextent_iter()
781 		rs->deferred++;  in xreap_rgextent_iter()
785 	trace_xreap_dispose_free_extent(rtg_group(sc->sr.rtg), rgbno, *rglenp);  in xreap_rgextent_iter()
794 	xfs_refcount_free_cow_extent(sc->tp, true, rtbno, *rglenp);  in xreap_rgextent_iter()
795 	error = xfs_free_extent_later(sc->tp, rtbno, *rglenp, NULL,  in xreap_rgextent_iter()
796 			rs->resv,  in xreap_rgextent_iter()
802 	rs->deferred++;  in xreap_rgextent_iter()
811  * Break a rt file metadata extent into sub-extents by fate (crosslinked, not
812  * crosslinked), and dispose of each sub-extent separately.  The extent must
821 	struct xreap_state	*rs = priv;  in xreap_rtmeta_extent()  local
822 	struct xfs_scrub	*sc = rs->sc;  in xreap_rtmeta_extent()
823 	xfs_rgblock_t		rgbno = xfs_rtb_to_rgbno(sc->mp, rtbno);  in xreap_rtmeta_extent()
827 	ASSERT(sc->ip != NULL);  in xreap_rtmeta_extent()
828 	ASSERT(!sc->sr.rtg);  in xreap_rtmeta_extent()
834 	sc->sr.rtg = xfs_rtgroup_get(sc->mp, xfs_rtb_to_rgno(sc->mp, rtbno));  in xreap_rtmeta_extent()
835 	if (!sc->sr.rtg)  in xreap_rtmeta_extent()
836 		return -EFSCORRUPTED;  in xreap_rtmeta_extent()
838 	xfs_rtgroup_lock(sc->sr.rtg, XREAP_RTGLOCK_ALL);  in xreap_rtmeta_extent()
844 		error = xreap_rgextent_select(rs, rgbno, rgbno_next,  in xreap_rtmeta_extent()
849 		error = xreap_rgextent_iter(rs, rgbno, &rglen, crosslinked);  in xreap_rtmeta_extent()
853 		if (xreap_want_defer_finish(rs)) {  in xreap_rtmeta_extent()
854 			error = xfs_defer_finish(&sc->tp);  in xreap_rtmeta_extent()
857 			xreap_defer_finish_reset(rs);  in xreap_rtmeta_extent()
858 		} else if (xreap_want_roll(rs)) {  in xreap_rtmeta_extent()
859 			error = xfs_trans_roll_inode(&sc->tp, sc->ip);  in xreap_rtmeta_extent()
862 			xreap_reset(rs);  in xreap_rtmeta_extent()
869 	xfs_rtgroup_unlock(sc->sr.rtg, XREAP_RTGLOCK_ALL);  in xreap_rtmeta_extent()
870 	xfs_rtgroup_put(sc->sr.rtg);  in xreap_rtmeta_extent()
871 	sc->sr.rtg = NULL;  in xreap_rtmeta_extent()
885 	struct xreap_state		rs = {  in xrep_reap_rtblocks()  local
892 	ASSERT(xfs_has_rmapbt(sc->mp));  in xrep_reap_rtblocks()
893 	ASSERT(sc->ip != NULL);  in xrep_reap_rtblocks()
895 	error = xrtb_bitmap_walk(bitmap, xreap_rtmeta_extent, &rs);  in xrep_reap_rtblocks()
899 	if (xreap_dirty(&rs))  in xrep_reap_rtblocks()
921 	struct xreap_state		rs = {  in xrep_reap_metadir_fsblocks()  local
928 	ASSERT(xfs_has_rmapbt(sc->mp));  in xrep_reap_metadir_fsblocks()
929 	ASSERT(sc->ip != NULL);  in xrep_reap_metadir_fsblocks()
930 	ASSERT(xfs_is_metadir_inode(sc->ip));  in xrep_reap_metadir_fsblocks()
932 	xfs_rmap_ino_bmbt_owner(&oinfo, sc->ip->i_ino, XFS_DATA_FORK);  in xrep_reap_metadir_fsblocks()
934 	error = xfsb_bitmap_walk(bitmap, xreap_fsmeta_extent, &rs);  in xrep_reap_metadir_fsblocks()
938 	if (xreap_dirty(&rs))  in xrep_reap_metadir_fsblocks()
949  * This first step determines the longest subset of the passed-in imap
969 	agbno = XFS_FSB_TO_AGBNO(sc->mp, imap->br_startblock);  in xreap_bmapi_select()
970 	agbno_next = agbno + imap->br_blockcount;  in xreap_bmapi_select()
972 	cur = xfs_rmapbt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp,  in xreap_bmapi_select()
973 			sc->sa.pag);  in xreap_bmapi_select()
975 	xfs_rmap_ino_owner(&oinfo, ip->i_ino, whichfork, imap->br_startoff);  in xreap_bmapi_select()
997 	imap->br_blockcount = len;  in xreap_bmapi_select()
998 	trace_xreap_bmapi_select(pag_group(sc->sa.pag), agbno, len,  in xreap_bmapi_select()
1018 	for (i = 0; i < bp->b_map_count; i++) {  in xreap_buf_loggable()
1022 		chunks = DIV_ROUND_UP(BBTOB(bp->b_maps[i].bm_len),  in xreap_buf_loggable()
1043 	struct xfs_mount	*mp = sc->mp;  in xreap_bmapi_binval()
1044 	struct xfs_perag	*pag = sc->sa.pag;  in xreap_bmapi_binval()
1056 	 * Avoid invalidating AG headers and post-EOFS blocks because we never  in xreap_bmapi_binval()
1059 	agbno = bno = XFS_FSB_TO_AGBNO(sc->mp, imap->br_startblock);  in xreap_bmapi_binval()
1060 	agbno_next = agbno + imap->br_blockcount;  in xreap_bmapi_binval()
1062 	    !xfs_verify_agbno(pag, agbno_next - 1))  in xreap_bmapi_binval()
1073 	off = imap->br_startoff + imap->br_blockcount;  in xreap_bmapi_binval()
1079 		error = xfs_bmapi_read(ip, off, max_off - off, &hmap,  in xreap_bmapi_binval()
1085 			return -EFSCORRUPTED;  in xreap_bmapi_binval()
1093 	scan_blocks = off - imap->br_startoff;  in xreap_bmapi_binval()
1114 				xfs_trans_bjoin(sc->tp, bp);  in xreap_bmapi_binval()
1115 				xfs_trans_binval(sc->tp, bp);  in xreap_bmapi_binval()
1128 				imap->br_blockcount = agbno_next - bno;  in xreap_bmapi_binval()
1134 		scan_blocks--;  in xreap_bmapi_binval()
1138 	trace_xreap_bmapi_binval(pag_group(sc->sa.pag), agbno,  in xreap_bmapi_binval()
1139 			imap->br_blockcount);  in xreap_bmapi_binval()
1145  * The number of blocks disposed of is returned in @imap->br_blockcount.
1166 		trace_xreap_dispose_unmap_extent(pag_group(sc->sa.pag),  in xrep_reap_bmapi_iter()
1167 				XFS_FSB_TO_AGBNO(sc->mp, imap->br_startblock),  in xrep_reap_bmapi_iter()
1168 				imap->br_blockcount);  in xrep_reap_bmapi_iter()
1175 		xfs_bmap_unmap_extent(sc->tp, ip, whichfork, imap);  in xrep_reap_bmapi_iter()
1176 		xfs_trans_mod_dquot_byino(sc->tp, ip, XFS_TRANS_DQ_BCOUNT,  in xrep_reap_bmapi_iter()
1177 				-(int64_t)imap->br_blockcount);  in xrep_reap_bmapi_iter()
1178 		xfs_rmap_unmap_extent(sc->tp, ip, whichfork, imap);  in xrep_reap_bmapi_iter()
1189 	trace_xreap_dispose_free_extent(pag_group(sc->sa.pag),  in xrep_reap_bmapi_iter()
1190 			XFS_FSB_TO_AGBNO(sc->mp, imap->br_startblock),  in xrep_reap_bmapi_iter()
1191 			imap->br_blockcount);  in xrep_reap_bmapi_iter()
1200 	if (error || imap->br_blockcount == 0)  in xrep_reap_bmapi_iter()
1208 	xfs_bmap_unmap_extent(sc->tp, ip, whichfork, imap);  in xrep_reap_bmapi_iter()
1209 	xfs_trans_mod_dquot_byino(sc->tp, ip, XFS_TRANS_DQ_BCOUNT,  in xrep_reap_bmapi_iter()
1210 			-(int64_t)imap->br_blockcount);  in xrep_reap_bmapi_iter()
1211 	return xfs_free_extent_later(sc->tp, imap->br_startblock,  in xrep_reap_bmapi_iter()
1212 			imap->br_blockcount, NULL, XFS_AG_RESV_NONE,  in xrep_reap_bmapi_iter()
1231 	ASSERT(sc->sa.pag == NULL);  in xreap_ifork_extent()
1235 	agno = XFS_FSB_TO_AGNO(sc->mp, imap->br_startblock);  in xreap_ifork_extent()
1236 	sc->sa.pag = xfs_perag_get(sc->mp, agno);  in xreap_ifork_extent()
1237 	if (!sc->sa.pag)  in xreap_ifork_extent()
1238 		return -EFSCORRUPTED;  in xreap_ifork_extent()
1240 	error = xfs_alloc_read_agf(sc->sa.pag, sc->tp, 0, &sc->sa.agf_bp);  in xreap_ifork_extent()
1257 	xfs_trans_brelse(sc->tp, sc->sa.agf_bp);  in xreap_ifork_extent()
1258 	sc->sa.agf_bp = NULL;  in xreap_ifork_extent()
1260 	xfs_perag_put(sc->sa.pag);  in xreap_ifork_extent()
1261 	sc->sa.pag = NULL;  in xreap_ifork_extent()
1267  * must hold ILOCK_EXCL, and ip can only be sc->ip or sc->tempip.  The fork
1280 	ASSERT(xfs_has_rmapbt(sc->mp));  in xrep_reap_ifork()
1281 	ASSERT(ip == sc->ip || ip == sc->tempip);  in xrep_reap_ifork()
1289 		error = xfs_bmapi_read(ip, off, XFS_MAX_FILEOFF - off, &imap,  in xrep_reap_ifork()
1295 			return -EFSCORRUPTED;  in xrep_reap_ifork()
1307 			error = xfs_defer_finish(&sc->tp);  in xrep_reap_ifork()