1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright 2017, Anshuman Khandual, IBM Corp.
4 *
5 * Works on architectures which support 128TB virtual
6 * address range and beyond.
7 */
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <string.h>
11 #include <unistd.h>
12 #include <errno.h>
13 #include <sys/prctl.h>
14 #include <sys/mman.h>
15 #include <sys/time.h>
16 #include <fcntl.h>
17
18 #include "vm_util.h"
19 #include "../kselftest.h"
20
21 /*
22 * Maximum address range mapped with a single mmap()
23 * call is little bit more than 1GB. Hence 1GB is
24 * chosen as the single chunk size for address space
25 * mapping.
26 */
27
28 #define SZ_1GB (1024 * 1024 * 1024UL)
29 #define SZ_1TB (1024 * 1024 * 1024 * 1024UL)
30
31 #define MAP_CHUNK_SIZE SZ_1GB
32
33 /*
34 * Address space till 128TB is mapped without any hint
35 * and is enabled by default. Address space beyond 128TB
36 * till 512TB is obtained by passing hint address as the
37 * first argument into mmap() system call.
38 *
39 * The process heap address space is divided into two
40 * different areas one below 128TB and one above 128TB
41 * till it reaches 512TB. One with size 128TB and the
42 * other being 384TB.
43 *
44 * On Arm64 the address space is 256TB and support for
45 * high mappings up to 4PB virtual address space has
46 * been added.
47 */
48
49 #define NR_CHUNKS_128TB ((128 * SZ_1TB) / MAP_CHUNK_SIZE) /* Number of chunks for 128TB */
50 #define NR_CHUNKS_256TB (NR_CHUNKS_128TB * 2UL)
51 #define NR_CHUNKS_384TB (NR_CHUNKS_128TB * 3UL)
52 #define NR_CHUNKS_3840TB (NR_CHUNKS_128TB * 30UL)
53
54 #define ADDR_MARK_128TB (1UL << 47) /* First address beyond 128TB */
55 #define ADDR_MARK_256TB (1UL << 48) /* First address beyond 256TB */
56
57 #ifdef __aarch64__
58 #define HIGH_ADDR_MARK ADDR_MARK_256TB
59 #define HIGH_ADDR_SHIFT 49
60 #define NR_CHUNKS_LOW NR_CHUNKS_256TB
61 #define NR_CHUNKS_HIGH NR_CHUNKS_3840TB
62 #else
63 #define HIGH_ADDR_MARK ADDR_MARK_128TB
64 #define HIGH_ADDR_SHIFT 48
65 #define NR_CHUNKS_LOW NR_CHUNKS_128TB
66 #define NR_CHUNKS_HIGH NR_CHUNKS_384TB
67 #endif
68
hint_addr(void)69 static char *hint_addr(void)
70 {
71 int bits = HIGH_ADDR_SHIFT + rand() % (63 - HIGH_ADDR_SHIFT);
72
73 return (char *) (1UL << bits);
74 }
75
validate_addr(char * ptr,int high_addr)76 static void validate_addr(char *ptr, int high_addr)
77 {
78 unsigned long addr = (unsigned long) ptr;
79
80 if (high_addr && addr < HIGH_ADDR_MARK)
81 ksft_exit_fail_msg("Bad address %lx\n", addr);
82
83 if (addr > HIGH_ADDR_MARK)
84 ksft_exit_fail_msg("Bad address %lx\n", addr);
85 }
86
mark_range(char * ptr,size_t size)87 static void mark_range(char *ptr, size_t size)
88 {
89 if (prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ptr, size, "virtual_address_range") == -1) {
90 if (errno == EINVAL) {
91 /* Depends on CONFIG_ANON_VMA_NAME */
92 ksft_test_result_skip("prctl(PR_SET_VMA_ANON_NAME) not supported\n");
93 ksft_finished();
94 } else {
95 ksft_exit_fail_perror("prctl(PR_SET_VMA_ANON_NAME) failed\n");
96 }
97 }
98 }
99
is_marked_vma(const char * vma_name)100 static int is_marked_vma(const char *vma_name)
101 {
102 return vma_name && !strcmp(vma_name, "[anon:virtual_address_range]\n");
103 }
104
validate_lower_address_hint(void)105 static int validate_lower_address_hint(void)
106 {
107 char *ptr;
108
109 ptr = mmap((void *) (1UL << 45), MAP_CHUNK_SIZE, PROT_READ |
110 PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
111
112 if (ptr == MAP_FAILED)
113 return 0;
114
115 return 1;
116 }
117
validate_complete_va_space(void)118 static int validate_complete_va_space(void)
119 {
120 unsigned long start_addr, end_addr, prev_end_addr;
121 char line[400];
122 char prot[6];
123 FILE *file;
124 int fd;
125
126 fd = open("va_dump", O_CREAT | O_WRONLY, 0600);
127 unlink("va_dump");
128 if (fd < 0) {
129 ksft_test_result_skip("cannot create or open dump file\n");
130 ksft_finished();
131 }
132
133 file = fopen("/proc/self/maps", "r");
134 if (file == NULL)
135 ksft_exit_fail_msg("cannot open /proc/self/maps\n");
136
137 prev_end_addr = 0;
138 while (fgets(line, sizeof(line), file)) {
139 const char *vma_name = NULL;
140 int vma_name_start = 0;
141 unsigned long hop;
142
143 if (sscanf(line, "%lx-%lx %4s %*s %*s %*s %n",
144 &start_addr, &end_addr, prot, &vma_name_start) != 3)
145 ksft_exit_fail_msg("cannot parse /proc/self/maps\n");
146
147 if (vma_name_start)
148 vma_name = line + vma_name_start;
149
150 /* end of userspace mappings; ignore vsyscall mapping */
151 if (start_addr & (1UL << 63))
152 return 0;
153
154 /* /proc/self/maps must have gaps less than MAP_CHUNK_SIZE */
155 if (start_addr - prev_end_addr >= MAP_CHUNK_SIZE)
156 return 1;
157
158 prev_end_addr = end_addr;
159
160 if (prot[0] != 'r')
161 continue;
162
163 if (check_vmflag_io((void *)start_addr))
164 continue;
165
166 /*
167 * Confirm whether MAP_CHUNK_SIZE chunk can be found or not.
168 * If write succeeds, no need to check MAP_CHUNK_SIZE - 1
169 * addresses after that. If the address was not held by this
170 * process, write would fail with errno set to EFAULT.
171 * Anyways, if write returns anything apart from 1, exit the
172 * program since that would mean a bug in /proc/self/maps.
173 */
174 hop = 0;
175 while (start_addr + hop < end_addr) {
176 if (write(fd, (void *)(start_addr + hop), 1) != 1)
177 return 1;
178 lseek(fd, 0, SEEK_SET);
179
180 if (is_marked_vma(vma_name))
181 munmap((char *)(start_addr + hop), MAP_CHUNK_SIZE);
182
183 hop += MAP_CHUNK_SIZE;
184 }
185 }
186 return 0;
187 }
188
main(int argc,char * argv[])189 int main(int argc, char *argv[])
190 {
191 char *ptr[NR_CHUNKS_LOW];
192 char **hptr;
193 char *hint;
194 unsigned long i, lchunks, hchunks;
195
196 ksft_print_header();
197 ksft_set_plan(1);
198
199 for (i = 0; i < NR_CHUNKS_LOW; i++) {
200 ptr[i] = mmap(NULL, MAP_CHUNK_SIZE, PROT_READ,
201 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
202
203 if (ptr[i] == MAP_FAILED) {
204 if (validate_lower_address_hint())
205 ksft_exit_fail_msg("mmap unexpectedly succeeded with hint\n");
206 break;
207 }
208
209 mark_range(ptr[i], MAP_CHUNK_SIZE);
210 validate_addr(ptr[i], 0);
211 }
212 lchunks = i;
213 hptr = (char **) calloc(NR_CHUNKS_HIGH, sizeof(char *));
214 if (hptr == NULL) {
215 ksft_test_result_skip("Memory constraint not fulfilled\n");
216 ksft_finished();
217 }
218
219 for (i = 0; i < NR_CHUNKS_HIGH; i++) {
220 hint = hint_addr();
221 hptr[i] = mmap(hint, MAP_CHUNK_SIZE, PROT_READ,
222 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
223
224 if (hptr[i] == MAP_FAILED)
225 break;
226
227 mark_range(ptr[i], MAP_CHUNK_SIZE);
228 validate_addr(hptr[i], 1);
229 }
230 hchunks = i;
231 if (validate_complete_va_space()) {
232 ksft_test_result_fail("BUG in mmap() or /proc/self/maps\n");
233 ksft_finished();
234 }
235
236 for (i = 0; i < lchunks; i++)
237 munmap(ptr[i], MAP_CHUNK_SIZE);
238
239 for (i = 0; i < hchunks; i++)
240 munmap(hptr[i], MAP_CHUNK_SIZE);
241
242 free(hptr);
243
244 ksft_test_result_pass("Test\n");
245 ksft_finished();
246 }
247