1 // SPDX-License-Identifier: GPL-2.0
7  * Laboratoire MASI - Institut Blaise Pascal
38 #include <linux/percpu-rwsem.h>
61  * with DOUBLE_CHECK defined mballoc creates persistent in-core
77 		printk(KERN_DEBUG "EXT4-fs DEBUG (%s, %d): %s:",	\
95 	pr_debug("[%s/%d] EXT4-fs (%s): ino %lu: (%s, %d): %s:" fmt,	\
96 		 current->comm, task_pid_nr(current),			\
97 		 ino->i_sb->s_id, ino->i_ino, __FILE__, __LINE__,	\
116 /* data type for filesystem-wide blocks number */
146 	 * Tries to lookup in-memory data structures to find the most
206 /* blocks already pre-reserved by delayed allocation */
279  * Flags for ext4_io_end->flags
294 	struct list_head	list;		/* per-file finished IO list */
319 #define EXT4_BOOT_LOADER_INO	 5	/* Boot loader inode */
324 /* First non-reserved inode for old ext4 filesystems */
333  * Macro-instructions used to manage several block sizes
341 # define EXT4_BLOCK_SIZE(s)		((s)->s_blocksize)
343 # define EXT4_BLOCK_SIZE(s)		(EXT4_MIN_BLOCK_SIZE << (s)->s_log_block_size)
347 					 EXT4_SB(s)->s_cluster_bits)
349 # define EXT4_BLOCK_SIZE_BITS(s)	((s)->s_blocksize_bits)
350 # define EXT4_CLUSTER_BITS(s)		(EXT4_SB(s)->s_cluster_bits)
352 # define EXT4_BLOCK_SIZE_BITS(s)	((s)->s_log_block_size + 10)
355 #define	EXT4_ADDR_PER_BLOCK_BITS(s)	(EXT4_SB(s)->s_addr_per_block_bits)
356 #define EXT4_INODE_SIZE(s)		(EXT4_SB(s)->s_inode_size)
357 #define EXT4_FIRST_INO(s)		(EXT4_SB(s)->s_first_ino)
359 #define EXT4_INODE_SIZE(s)	(((s)->s_rev_level == EXT4_GOOD_OLD_REV) ? \
361 				 (s)->s_inode_size)
362 #define EXT4_FIRST_INO(s)	(((s)->s_rev_level == EXT4_GOOD_OLD_REV) ? \
364 				 (s)->s_first_ino)
368 	((EXT4_BLOCK_ALIGN(size + offset, blkbits) >> blkbits) - (offset >> \
372 #define EXT4_B2C(sbi, blk)	((blk) >> (sbi)->s_cluster_bits)
374 #define EXT4_C2B(sbi, cluster)	((cluster) << (sbi)->s_cluster_bits)
375 /* Translate # of blks to # of clusters */
376 #define EXT4_NUM_B2C(sbi, blks)	(((blks) + (sbi)->s_cluster_ratio - 1) >> \  argument
377 				 (sbi)->s_cluster_bits)
380 				  ~((ext4_fsblk_t) (s)->s_cluster_ratio - 1))
382 				  ~((ext4_lblk_t) (s)->s_cluster_ratio - 1))
385 				    ((ext4_lblk_t) (sbi)->s_cluster_ratio - 1))
388 				 ((ext4_fsblk_t) (s)->s_cluster_ratio - 1))
390 				 ((ext4_lblk_t) (s)->s_cluster_ratio - 1))
441 #define EXT4_BG_INODE_ZEROED	0x0004 /* On-disk itable initialized to zero */
444  * Macro-instructions used to manage group descriptors
449 #define EXT4_DESC_SIZE(s)		(EXT4_SB(s)->s_desc_size)
451 # define EXT4_BLOCKS_PER_GROUP(s)	(EXT4_SB(s)->s_blocks_per_group)
452 # define EXT4_CLUSTERS_PER_GROUP(s)	(EXT4_SB(s)->s_clusters_per_group)
453 # define EXT4_DESC_PER_BLOCK(s)		(EXT4_SB(s)->s_desc_per_block)
454 # define EXT4_INODES_PER_GROUP(s)	(EXT4_SB(s)->s_inodes_per_group)
455 # define EXT4_DESC_PER_BLOCK_BITS(s)	(EXT4_SB(s)->s_desc_per_block_bits)
457 # define EXT4_BLOCKS_PER_GROUP(s)	((s)->s_blocks_per_group)
459 # define EXT4_INODES_PER_GROUP(s)	((s)->s_inodes_per_group)
488 /* End compression flags --- maybe not all used */
489 #define EXT4_INDEX_FL			0x00001000 /* hash-indexed directory */
545 /* Flags that are appropriate for regular files (all but dir-specific ones). */
549 /* Flags that are appropriate for non-directories/regular files. */
587 /* End compression flags --- maybe not all used */
588 	EXT4_INODE_INDEX	= 12,	/* hash-indexed directory */
608  * build-time check to make sure that EXT4_XXX_FL is consistent with respect to
613  * with the values of FS_XXX_FL defined in include/linux/fs.h and the on-disk
617  * It's not paranoia if the Murphy's Law really *is* out to get you.  :-)
828 	__le32  i_version_hi;	/* high 32 bits for 64-bit version */
833 #define EXT4_EPOCH_MASK ((1 << EXT4_EPOCH_BITS) - 1)
838  * with large inodes (-I 256 or larger) and there are not currently any EAs
844  * inode-size = GOOD_OLD_INODE_SIZE + i_extra_isize
848 	  sizeof((ext4_inode)->field))			\
850 	    (einode)->i_extra_isize))			\
856  * epoch  32-bit                         32-bit tv_sec to
857  * bits   time    decoded 64-bit tv_sec  64-bit tv_sec      valid time range
858  * 0 0    1    -0x80000000..-0x00000001  0x000000000 1901-12-13..1969-12-31
859  * 0 0    0    0x000000000..0x07fffffff  0x000000000 1970-01-01..2038-01-19
860  * 0 1    1    0x080000000..0x0ffffffff  0x100000000 2038-01-19..2106-02-07
861  * 0 1    0    0x100000000..0x17fffffff  0x100000000 2106-02-07..2174-02-25
862  * 1 0    1    0x180000000..0x1ffffffff  0x200000000 2174-02-25..2242-03-16
863  * 1 0    0    0x200000000..0x27fffffff  0x200000000 2242-03-16..2310-04-04
864  * 1 1    1    0x280000000..0x2ffffffff  0x300000000 2310-04-04..2378-04-22
865  * 1 1    0    0x300000000..0x37fffffff  0x300000000 2378-04-22..2446-05-10
867  * Note that previous versions of the kernel on 64-bit systems would
875 	u32 extra = ((ts.tv_sec - (s32)ts.tv_sec) >> 32) & EXT4_EPOCH_MASK;  in ext4_encode_extra_time()
893 		(raw_inode)->xtime = cpu_to_le32((ts).tv_sec);			\
894 		(raw_inode)->xtime ## _extra = ext4_encode_extra_time(ts);	\
896 		(raw_inode)->xtime = cpu_to_le32(clamp_t(int32_t, (ts).tv_sec, S32_MIN, S32_MAX));	\
910 		EXT4_INODE_SET_XTIME_VAL(xtime, &((einode)->vfs_inode),		\
911 					 raw_inode, (einode)->xtime)
915 		ext4_decode_extra_time((raw_inode)->xtime,				\
916 				       (raw_inode)->xtime ## _extra) :		\
918 			.tv_sec = (signed)le32_to_cpu((raw_inode)->xtime)	\
942 		(einode)->xtime =						\
943 			EXT4_INODE_GET_XTIME_VAL(xtime, &(einode->vfs_inode),	\
946 		(einode)->xtime = (struct timespec64){0, 0};			\
984  * holding i_data_sem for a normal (non-quota) inode.  Since we don't
989  *  I_DATA_SEM_NORMAL - Used for most inodes
990  *  I_DATA_SEM_OTHER  - Used by move_inode.c for the second normal inode
993  *  I_DATA_SEM_QUOTA  - Used for quota inodes only
994  *  I_DATA_SEM_EA     - Used for ea_inodes only
1015 	 * it is used for making block allocation decisions - we try to
1050 					 * protected by sbi->s_fc_lock.
1078 	 * into the on-disk inode when writing inodes out, instead of i_size.
1093 	 * during recovery.  Hence we must fix the get_block-vs-truncate race
1133 	/* on-disk additional length */
1210 #define EXT4_MOUNT_NO_UID32		0x02000  /* Disable 32-bit UIDs */
1245 #define EXT4_MOUNT2_HURD_COMPAT		0x00000004 /* Support HURD-castrated
1258 #define clear_opt(sb, opt)		EXT4_SB(sb)->s_mount_opt &= \
1260 #define set_opt(sb, opt)		EXT4_SB(sb)->s_mount_opt |= \
1262 #define test_opt(sb, opt)		(EXT4_SB(sb)->s_mount_opt & \
1265 #define clear_opt2(sb, opt)		EXT4_SB(sb)->s_mount_opt2 &= \
1267 #define set_opt2(sb, opt)		EXT4_SB(sb)->s_mount_opt2 |= \
1269 #define test_opt2(sb, opt)		(EXT4_SB(sb)->s_mount_opt2 & \
1292 #define EXT4_ERRORS_RO			2	/* Remount fs read-only */
1343 	__le32	s_first_ino;		/* First non-reserved inode */
1348 	__le32	s_feature_ro_compat;	/* readonly-compatible feature set */
1349 /*68*/	__u8	s_uuid[16];		/* 128-bit uuid for volume */
1384 	__le64  s_mmp_block;            /* Block for multi-mount protection */
1396 					   on-disk snapshot list */
1435 #define EXT4_S_ERR_LEN (EXT4_S_ERR_END - EXT4_S_ERR_START)
1473 	return (sb->s_blocksize - sizeof(struct ext4_orphan_block_tail)) /  in ext4_inodes_per_orphan_block()
1493  * fourth extended-fs super-block data in memory
1505 	ext4_group_t s_blockfile_groups;/* Groups acceptable for non-extent files */
1603 	/* where last allocation was done - for stream allocation */
1641 	/* the size of zero-out chunk */
1687 	/* Encryption policy for '-o test_dummy_encryption' */
1721 	 * If we are in a context where we cannot update the on-disk
1750 	 * ei->i_fc_list, s_fc_dentry_q, s_fc_q, s_fc_bytes, s_fc_bh.
1764 	return sb->s_fs_info;  in EXT4_SB()
1773 	percpu_down_read(&EXT4_SB(sb)->s_writepages_rwsem);  in ext4_writepages_down_read()
1780 	percpu_up_read(&EXT4_SB(sb)->s_writepages_rwsem);  in ext4_writepages_up_read()
1785 	percpu_down_write(&EXT4_SB(sb)->s_writepages_rwsem);  in ext4_writepages_down_write()
1792 	percpu_up_write(&EXT4_SB(sb)->s_writepages_rwsem);  in ext4_writepages_up_write()
1799 		 ino <= le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count));  in ext4_valid_inum()
1803  * Returns: sbi->field[index]
1806  * - s_group_desc
1807  * - s_group_info
1808  * - s_flex_group
1812 	typeof(*((sbi)->field)) _v;					   \
1814 	_v = ((typeof(_v)*)rcu_dereference((sbi)->field))[index];	   \
1820  * run-time mount flags
1830 	set_bit(bit, &EXT4_SB(sb)->s_mount_flags);  in ext4_set_mount_flag()
1835 	clear_bit(bit, &EXT4_SB(sb)->s_mount_flags);  in ext4_clear_mount_flag()
1840 	return test_bit(bit, &EXT4_SB(sb)->s_mount_flags);  in ext4_test_mount_flag()
1862 	if (unlikely(sbi->s_simulate_fail == code)) {  in ext4_simulate_fail()
1863 		sbi->s_simulate_fail = 0;  in ext4_simulate_fail()
1901 	EXT4_STATE_XATTR,		/* has in-inode xattrs */
1903 	EXT4_STATE_DA_ALLOC_CLOSE,	/* Alloc DA blks on close */
1906 	EXT4_STATE_MAY_INLINE_DATA,	/* may have in-inode data */
1908 	EXT4_STATE_LUSTRE_EA_INODE,	/* Lustre-style ea_inode */
1909 	EXT4_STATE_VERITY_IN_PROGRESS,	/* building fs-verity Merkle tree */
1917 	return test_bit(bit + (offset), &EXT4_I(inode)->i_##field);	\
1921 	set_bit(bit + (offset), &EXT4_I(inode)->i_##field);		\
1925 	clear_bit(bit + (offset), &EXT4_I(inode)->i_##field);		\
1945 	(ei)->i_state_flags = 0;  in ext4_clear_state_flags()
1957  * a kernel struct super_block.  This will allow us to call the feature-test
1968 #define NEXT_ORPHAN(inode) EXT4_I(inode)->i_dtime
1989 #define EXT4_EXTRA_TIMESTAMP_MAX	(((s64)1 << 34) - 1  + S32_MIN)
2035 							   non-empty */
2049 #define EXT4_FEATURE_INCOMPAT_LARGEDIR		0x4000 /* >2GB or 3-lvl htree */
2059 	return ((EXT4_SB(sb)->s_es->s_feature_compat & \
2065 	EXT4_SB(sb)->s_es->s_feature_compat |= \
2070 	EXT4_SB(sb)->s_es->s_feature_compat &= \
2077 	return ((EXT4_SB(sb)->s_es->s_feature_ro_compat & \
2083 	EXT4_SB(sb)->s_es->s_feature_ro_compat |= \
2088 	EXT4_SB(sb)->s_es->s_feature_ro_compat &= \
2095 	return ((EXT4_SB(sb)->s_es->s_feature_incompat & \
2101 	EXT4_SB(sb)->s_es->s_feature_incompat |= \
2106 	EXT4_SB(sb)->s_es->s_feature_incompat &= \
2200 	return ((EXT4_SB(sb)->s_es->s_feature_compat & \  in EXT4_FEATURE_COMPAT_FUNCS()
2205 	return ((EXT4_SB(sb)->s_es->s_feature_ro_compat & \
2210 	return ((EXT4_SB(sb)->s_es->s_feature_incompat & \
2220 	return (EXT4_SB(sb)->s_es->s_feature_compat != 0);
2224 	return (EXT4_SB(sb)->s_es->s_feature_ro_compat != 0);  in ext4_has_ro_compat_features()
2228 	return (EXT4_SB(sb)->s_es->s_feature_incompat != 0);  in ext4_has_incompat_features()
2240 	EXT4_FLAGS_EMERGENCY_RO,/* Emergency read-only due to fs errors */
2245 	return test_bit(EXT4_FLAGS_SHUTDOWN, &EXT4_SB(sb)->s_ext4_flags);  in ext4_forced_shutdown()
2250 	return test_bit(EXT4_FLAGS_EMERGENCY_RO, &EXT4_SB(sb)->s_ext4_flags);  in ext4_emergency_ro()
2302 #define EXT4_BASE_DIR_LEN (sizeof(struct ext4_dir_entry_2) - EXT4_NAME_LEN)
2342 		((8 + (entry)->name_len + EXT4_DIR_ROUND) & ~EXT4_DIR_ROUND)))
2343 #define EXT4_DIRENT_HASH(entry) le32_to_cpu(EXT4_DIRENT_HASHES(entry)->hash)
2345 		le32_to_cpu(EXT4_DIRENT_HASHES(entry)->minor_hash)
2366 					((blocksize) - \
2392 #define EXT4_DIR_ROUND			(EXT4_DIR_PAD - 1)
2393 #define EXT4_MAX_REC_LEN		((1<<16)-1)
2452 #define is_dx(dir) (ext4_has_feature_dir_index((dir)->i_sb) && \
2454 #define EXT4_DIR_LINK_MAX(dir) unlikely((dir)->i_nlink >= EXT4_LINK_MAX && \
2455 		    !(ext4_has_feature_dir_nlink((dir)->i_sb) && is_dx(dir)))
2456 #define EXT4_DIR_LINK_EMPTY(dir) ((dir)->i_nlink == 2 || (dir)->i_nlink == 1)
2488 #define EXT4_HTREE_EOF_32BIT   ((1UL  << (32 - 1)) - 1)
2489 #define EXT4_HTREE_EOF_64BIT   ((1ULL << (64 - 1)) - 1)
2509 #define fname_name(p) ((p)->disk_name.name)
2510 #define fname_usr_name(p) ((p)->usr_fname->name)
2511 #define fname_len(p)  ((p)->disk_name.len)
2525 	return (struct ext4_inode *) (iloc->bh->b_data + iloc->offset);  in ext4_raw_inode()
2531 	       !(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL);  in ext4_is_quota_file()
2556 		le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block);  in ext4_group_first_block_no()
2562 #define ERR_BAD_DX_DIR	(-(MAX_ERRNO - 1))
2746 	kfree(fname->cf_name.name);  in ext4_fname_free_ci_filename()
2747 	fname->cf_name.name = NULL;  in ext4_fname_free_ci_filename()
2782 	fname->usr_fname = iname;  in ext4_fname_setup_filename()
2783 	fname->disk_name.name = (unsigned char *) iname->name;  in ext4_fname_setup_filename()
2784 	fname->disk_name.len = iname->len;  in ext4_fname_setup_filename()
2793 	return ext4_fname_setup_filename(dir, &dentry->d_name, 1, fname);  in ext4_fname_prepare_lookup()
2804 	return -EOPNOTSUPP;  in ext4_ioctl_get_encryption_pwsalt()
2833 	if (!ext4_has_feature_dir_index(inode->i_sb) &&  in ext4_update_dx_flag()
2836 		WARN_ON_ONCE(ext4_has_feature_metadata_csum(inode->i_sb));  in ext4_update_dx_flag()
3282 	(((es)->s_feature_incompat & cpu_to_le32(EXT4_FEATURE_INCOMPAT_64BIT) \
3283 		? (ext4_fsblk_t)le32_to_cpu(es->name##_hi) << 32 : 0) | \
3284 		le32_to_cpu(es->name##_lo))
3304 	es->s_blocks_count_lo = cpu_to_le32((u32)blk);  in ext4_blocks_count_set()
3305 	es->s_blocks_count_hi = cpu_to_le32(blk >> 32);  in ext4_blocks_count_set()
3311 	es->s_free_blocks_count_lo = cpu_to_le32((u32)blk);  in ext4_free_blocks_count_set()
3312 	es->s_free_blocks_count_hi = cpu_to_le32(blk >> 32);  in ext4_free_blocks_count_set()
3318 	es->s_r_blocks_count_lo = cpu_to_le32((u32)blk);  in ext4_r_blocks_count_set()
3319 	es->s_r_blocks_count_hi = cpu_to_le32(blk >> 32);  in ext4_r_blocks_count_set()
3326 	    S_ISREG(le16_to_cpu(raw_inode->i_mode)))  in ext4_isize()
3327 		return ((loff_t)le32_to_cpu(raw_inode->i_size_high) << 32) |  in ext4_isize()
3328 			le32_to_cpu(raw_inode->i_size_lo);  in ext4_isize()
3330 	return (loff_t) le32_to_cpu(raw_inode->i_size_lo);  in ext4_isize()
3335 	raw_inode->i_size_lo = cpu_to_le32(i_size);  in ext4_isize_set()
3336 	raw_inode->i_size_high = cpu_to_le32(i_size >> 32);  in ext4_isize_set()
3346 	ext4_group_t	ngroups = EXT4_SB(sb)->s_groups_count;  in ext4_get_groups_count()
3355 	return block_group >> sbi->s_log_groups_per_flex;  in ext4_flex_group()
3360 	return 1 << sbi->s_log_groups_per_flex;  in ext4_flex_bg_size()
3382 	WARN_ON_ONCE(S_ISREG(inode->i_mode) &&  in ext4_update_i_disksize()
3384 	down_write(&EXT4_I(inode)->i_data_sem);  in ext4_update_i_disksize()
3385 	if (newsize > EXT4_I(inode)->i_disksize)  in ext4_update_i_disksize()
3386 		WRITE_ONCE(EXT4_I(inode)->i_disksize, newsize);  in ext4_update_i_disksize()
3387 	up_write(&EXT4_I(inode)->i_data_sem);  in ext4_update_i_disksize()
3395 	if (newsize > inode->i_size) {  in ext4_update_inode_size()
3399 	if (newsize > EXT4_I(inode)->i_disksize) {  in ext4_update_inode_size()
3429 	ext4_grpblk_t	bb_counters[];	/* Nr of free power-of-two-block
3432 					 * 5 free 8-block regions. */
3446 	(test_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &((grp)->bb_state)))
3448 	(test_bit(EXT4_GROUP_INFO_BBITMAP_CORRUPT_BIT, &((grp)->bb_state)))
3450 	(test_bit(EXT4_GROUP_INFO_IBITMAP_CORRUPT_BIT, &((grp)->bb_state)))
3453 	(test_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state)))
3455 	(set_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state)))
3457 	(clear_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state)))
3459 	(test_and_set_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &((grp)->bb_state)))
3467 	return bgl_lock_ptr(EXT4_SB(sb)->s_blockgroup_lock, group);  in ext4_group_lock_ptr()
3476 	return (atomic_read(&sbi->s_lock_busy) > EXT4_CONTENTION_THRESHOLD);  in ext4_fs_is_busy()
3487 		atomic_add_unless(&EXT4_SB(sb)->s_lock_busy, -1, 0);  in ext4_lock_group()
3493 		atomic_add_unless(&EXT4_SB(sb)->s_lock_busy, 1,  in ext4_lock_group()
3516 		sbi->s_qf_names[USRQUOTA] || sbi->s_qf_names[GRPQUOTA]);  in ext4_is_quota_journalled()
3526 			    (__le32 *)(bh)->b_data,			\
3527 			    EXT4_ADDR_PER_BLOCK((inode)->i_sb))
3531 			    EXT4_I(inode)->i_data,			\
3602 	       EXT4_I(inode)->i_inline_off;  in ext4_has_inline_data()
3637 		de->file_type = ext4_type_by_mode[(mode & S_IFMT)>>S_SHIFT];  in ext4_set_de_type()
3742 /* page-io.c */
3783  * to mark the bitmap uptodate. We need to also zero-out the bitmap
3790 			test_bit(BH_BITMAP_UPTODATE, &(bh)->b_state));  in bitmap_uptodate()
3794 	set_bit(BH_BITMAP_UPTODATE, &(bh)->b_state);  in set_bitmap_uptodate()
3809 	if (!(io_end->flag & EXT4_IO_END_UNWRITTEN)) {  in ext4_set_io_unwritten_flag()
3810 		io_end->flag |= EXT4_IO_END_UNWRITTEN;  in ext4_set_io_unwritten_flag()
3811 		atomic_inc(&EXT4_I(inode)->i_unwritten);  in ext4_set_io_unwritten_flag()
3817 	struct inode *inode = io_end->inode;  in ext4_clear_io_unwritten_flag()
3819 	if (io_end->flag & EXT4_IO_END_UNWRITTEN) {  in ext4_clear_io_unwritten_flag()
3820 		io_end->flag &= ~EXT4_IO_END_UNWRITTEN;  in ext4_clear_io_unwritten_flag()
3822 		if (atomic_dec_and_test(&EXT4_I(inode)->i_unwritten))  in ext4_clear_io_unwritten_flag()
3847 	return S_ISREG(inode->i_mode) && EXT4_SB(inode->i_sb)->s_awu_min > 0;  in ext4_inode_can_atomic_write()