1 // SPDX-License-Identifier: LGPL-2.1
44 static __thread __attribute__((tls_model("initial-exec")))
49 static __thread __attribute__((tls_model("initial-exec"), unused))
113 	"ahi %%" INJECT_ASM_REG ", -1\n\t" \
204 	"addiu " INJECT_ASM_REG ", -1\n\t" \
226 	"addi  " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t"	\
250 	"l.addi  " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t"	\
269 	if (loc_nr_loops == -1 && opt_modulo) { \
270 		if (yield_mod_cnt == opt_modulo - 1) { \
326 int rseq_membarrier_expedited(int cpu)  in rseq_membarrier_expedited()  argument
351 int rseq_membarrier_expedited(int cpu)  in rseq_membarrier_expedited()  argument
354 			      MEMBARRIER_CMD_FLAG_CPU, cpu);  in rseq_membarrier_expedited()
412 	intptr_t offset;  member
429 	intptr_t offset;  member
438 /* A simple percpu spinlock. Grabs lock on current cpu. */
441 	int cpu;  in rseq_this_cpu_lock()  local
446 		cpu = get_current_cpu_id();  in rseq_this_cpu_lock()
447 		if (cpu < 0) {  in rseq_this_cpu_lock()
448 			fprintf(stderr, "pid: %d: tid: %d, cpu: %d: cid: %d\n",  in rseq_this_cpu_lock()
449 					getpid(), (int) rseq_gettid(), rseq_current_cpu_raw(), cpu);  in rseq_this_cpu_lock()
453 					 &lock->c[cpu].v,  in rseq_this_cpu_lock()
454 					 0, 1, cpu);  in rseq_this_cpu_lock()
464 	return cpu;  in rseq_this_cpu_lock()
467 static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu)  in rseq_percpu_unlock()  argument
469 	assert(lock->c[cpu].v == 1);  in rseq_percpu_unlock()
474 	rseq_smp_store_release(&lock->c[cpu].v, 0);  in rseq_percpu_unlock()
480 	struct spinlock_test_data *data = thread_data->data;  in test_percpu_spinlock_thread()
483 	if (!opt_disable_rseq && thread_data->reg &&  in test_percpu_spinlock_thread()
486 	reps = thread_data->reps;  in test_percpu_spinlock_thread()
488 		int cpu = rseq_this_cpu_lock(&data->lock);  in test_percpu_spinlock_thread()  local
489 		data->c[cpu].count++;  in test_percpu_spinlock_thread()
490 		rseq_percpu_unlock(&data->lock, cpu);  in test_percpu_spinlock_thread()
499 	if (!opt_disable_rseq && thread_data->reg &&  in test_percpu_spinlock_thread()
506  * A simple test which implements a sharded counter using a per-cpu
508  * per-cpu increment; however, this is reasonable for a test and the
557 	struct inc_test_data *data = thread_data->data;  in test_percpu_inc_thread()
560 	if (!opt_disable_rseq && thread_data->reg &&  in test_percpu_inc_thread()
563 	reps = thread_data->reps;  in test_percpu_inc_thread()
568 			int cpu;  in test_percpu_inc_thread()  local
570 			cpu = get_current_cpu_id();  in test_percpu_inc_thread()
572 					&data->c[cpu].count, 1, cpu);  in test_percpu_inc_thread()
582 	if (!opt_disable_rseq && thread_data->reg &&  in test_percpu_inc_thread()
635 	int cpu;  in this_cpu_list_push()  local
641 		cpu = get_current_cpu_id();  in this_cpu_list_push()
642 		/* Load list->c[cpu].head with single-copy atomicity. */  in this_cpu_list_push()
643 		expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head);  in this_cpu_list_push()
645 		targetptr = (intptr_t *)&list->c[cpu].head;  in this_cpu_list_push()
646 		node->next = (struct percpu_list_node *)expect;  in this_cpu_list_push()
648 					 targetptr, expect, newval, cpu);  in this_cpu_list_push()
654 		*_cpu = cpu;  in this_cpu_list_push()
658  * Unlike a traditional lock-less linked list; the availability of a
660  * ABA-type races.
666 	int cpu;  in this_cpu_list_pop()  local
671 		long offset;  in this_cpu_list_pop()  local
674 		cpu = get_current_cpu_id();  in this_cpu_list_pop()
675 		targetptr = (intptr_t *)&list->c[cpu].head;  in this_cpu_list_pop()
677 		offset = offsetof(struct percpu_list_node, next);  in this_cpu_list_pop()
681 						 offset, load, cpu);  in this_cpu_list_pop()
691 		*_cpu = cpu;  in this_cpu_list_pop()
699 struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu)  in __percpu_list_pop()  argument
703 	node = list->c[cpu].head;  in __percpu_list_pop()
706 	list->c[cpu].head = node->next;  in __percpu_list_pop()
737 /* Simultaneous modification to a per-cpu linked list from many threads.  */
749 	/* Generate list entries for every usable cpu. */  in test_percpu_list()
761 			node->data = j;  in test_percpu_list()
762 			node->next = list.c[i].head;  in test_percpu_list()
793 			sum += node->data;  in test_percpu_list()
811 	int cpu;  in this_cpu_buffer_push()  local
816 		intptr_t offset;  in this_cpu_buffer_push()  local
819 		cpu = get_current_cpu_id();  in this_cpu_buffer_push()
820 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_buffer_push()
821 		if (offset == buffer->c[cpu].buflen)  in this_cpu_buffer_push()
824 		targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset];  in this_cpu_buffer_push()
825 		newval_final = offset + 1;  in this_cpu_buffer_push()
826 		targetptr_final = &buffer->c[cpu].offset;  in this_cpu_buffer_push()
828 			targetptr_final, offset, targetptr_spec,  in this_cpu_buffer_push()
829 			newval_spec, newval_final, cpu);  in this_cpu_buffer_push()
837 		*_cpu = cpu;  in this_cpu_buffer_push()
845 	int cpu;  in this_cpu_buffer_pop()  local
849 		intptr_t offset;  in this_cpu_buffer_pop()  local
852 		cpu = get_current_cpu_id();  in this_cpu_buffer_pop()
853 		/* Load offset with single-copy atomicity. */  in this_cpu_buffer_pop()
854 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_buffer_pop()
855 		if (offset == 0) {  in this_cpu_buffer_pop()
859 		head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]);  in this_cpu_buffer_pop()
860 		newval = offset - 1;  in this_cpu_buffer_pop()
861 		targetptr = (intptr_t *)&buffer->c[cpu].offset;  in this_cpu_buffer_pop()
863 			targetptr, offset,  in this_cpu_buffer_pop()
864 			(intptr_t *)&buffer->c[cpu].array[offset - 1],  in this_cpu_buffer_pop()
865 			(intptr_t)head, newval, cpu);  in this_cpu_buffer_pop()
871 		*_cpu = cpu;  in this_cpu_buffer_pop()
880 					       int cpu)  in __percpu_buffer_pop()  argument
883 	intptr_t offset;  in __percpu_buffer_pop()  local
885 	offset = buffer->c[cpu].offset;  in __percpu_buffer_pop()
886 	if (offset == 0)  in __percpu_buffer_pop()
888 	head = buffer->c[cpu].array[offset - 1];  in __percpu_buffer_pop()
889 	buffer->c[cpu].offset = offset - 1;  in __percpu_buffer_pop()
924 /* Simultaneous modification to a per-cpu buffer from many threads.  */
936 	/* Generate list entries for every usable cpu. */  in test_percpu_buffer()
941 		/* Worse-case is every item in same CPU. */  in test_percpu_buffer()
953 			 * We could theoretically put the word-sized  in test_percpu_buffer()
961 			node->data = j;  in test_percpu_buffer()
962 			buffer.c[i].array[j - 1] = node;  in test_percpu_buffer()
963 			buffer.c[i].offset++;  in test_percpu_buffer()
993 			sum += node->data;  in test_percpu_buffer()
1012 	int cpu;  in this_cpu_memcpy_buffer_push()  local
1015 		intptr_t *targetptr_final, newval_final, offset;  in this_cpu_memcpy_buffer_push()  local
1020 		cpu = get_current_cpu_id();  in this_cpu_memcpy_buffer_push()
1021 		/* Load offset with single-copy atomicity. */  in this_cpu_memcpy_buffer_push()
1022 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_memcpy_buffer_push()
1023 		if (offset == buffer->c[cpu].buflen)  in this_cpu_memcpy_buffer_push()
1025 		destptr = (char *)&buffer->c[cpu].array[offset];  in this_cpu_memcpy_buffer_push()
1029 		newval_final = offset + 1;  in this_cpu_memcpy_buffer_push()
1030 		targetptr_final = &buffer->c[cpu].offset;  in this_cpu_memcpy_buffer_push()
1033 			targetptr_final, offset,  in this_cpu_memcpy_buffer_push()
1035 			newval_final, cpu);  in this_cpu_memcpy_buffer_push()
1043 		*_cpu = cpu;  in this_cpu_memcpy_buffer_push()
1052 	int cpu;  in this_cpu_memcpy_buffer_pop()  local
1055 		intptr_t *targetptr_final, newval_final, offset;  in this_cpu_memcpy_buffer_pop()  local
1060 		cpu = get_current_cpu_id();  in this_cpu_memcpy_buffer_pop()
1061 		/* Load offset with single-copy atomicity. */  in this_cpu_memcpy_buffer_pop()
1062 		offset = RSEQ_READ_ONCE(buffer->c[cpu].offset);  in this_cpu_memcpy_buffer_pop()
1063 		if (offset == 0)  in this_cpu_memcpy_buffer_pop()
1066 		srcptr = (char *)&buffer->c[cpu].array[offset - 1];  in this_cpu_memcpy_buffer_pop()
1069 		newval_final = offset - 1;  in this_cpu_memcpy_buffer_pop()
1070 		targetptr_final = &buffer->c[cpu].offset;  in this_cpu_memcpy_buffer_pop()
1072 			targetptr_final, offset, destptr, srcptr, copylen,  in this_cpu_memcpy_buffer_pop()
1073 			newval_final, cpu);  in this_cpu_memcpy_buffer_pop()
1081 		*_cpu = cpu;  in this_cpu_memcpy_buffer_pop()
1091 				int cpu)  in __percpu_memcpy_buffer_pop()  argument
1093 	intptr_t offset;  in __percpu_memcpy_buffer_pop()  local
1095 	offset = buffer->c[cpu].offset;  in __percpu_memcpy_buffer_pop()
1096 	if (offset == 0)  in __percpu_memcpy_buffer_pop()
1098 	memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item));  in __percpu_memcpy_buffer_pop()
1099 	buffer->c[cpu].offset = offset - 1;  in __percpu_memcpy_buffer_pop()
1135 /* Simultaneous modification to a per-cpu buffer from many threads.  */
1147 	/* Generate list entries for every usable cpu. */  in test_percpu_memcpy_buffer()
1152 		/* Worse-case is every item in same CPU. */  in test_percpu_memcpy_buffer()
1162 			 * We could theoretically put the word-sized  in test_percpu_memcpy_buffer()
1168 			buffer.c[i].array[j - 1].data1 = j;  in test_percpu_memcpy_buffer()
1169 			buffer.c[i].array[j - 1].data2 = j + 1;  in test_percpu_memcpy_buffer()
1170 			buffer.c[i].offset++;  in test_percpu_memcpy_buffer()
1268 	while (!__atomic_load_n(&args->percpu_list_ptr, __ATOMIC_ACQUIRE)) {}  in test_membarrier_worker_thread()
1274 			int cpu = get_current_cpu_id();  in test_membarrier_worker_thread()  local
1277 				&args->percpu_list_ptr,  in test_membarrier_worker_thread()
1278 				sizeof(struct percpu_list_entry) * cpu, 1, cpu);  in test_membarrier_worker_thread()
1300 		node->data = 0;  in test_membarrier_init_percpu_list()
1301 		node->next = NULL;  in test_membarrier_init_percpu_list()
1302 		list->c[i].head = node;  in test_membarrier_init_percpu_list()
1311 		free(list->c[i].head);  in test_membarrier_free_percpu_list()
1315  * The manager thread swaps per-cpu lists that worker threads see,
1336 	__atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE);  in test_membarrier_manager_thread()
1338 	while (!__atomic_load_n(&args->stop, __ATOMIC_ACQUIRE)) {  in test_membarrier_manager_thread()
1345 		if (expect_b != __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE)) {  in test_membarrier_manager_thread()
1351 		__atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_b, __ATOMIC_RELEASE);  in test_membarrier_manager_thread()
1353 				errno != ENXIO /* missing CPU */) {  in test_membarrier_manager_thread()
1358 		 * Cpu A should now only modify list_b, so the values  in test_membarrier_manager_thread()
1361 		expect_a = __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE);  in test_membarrier_manager_thread()
1368 		if (expect_a != __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE)) {  in test_membarrier_manager_thread()
1374 		__atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE);  in test_membarrier_manager_thread()
1376 				errno != ENXIO /* missing CPU*/) {  in test_membarrier_manager_thread()
1381 		expect_b = __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE);  in test_membarrier_manager_thread()
1459 	printf("	[-1 loops] Number of loops for delay injection 1\n");  in show_usage()
1460 	printf("	[-2 loops] Number of loops for delay injection 2\n");  in show_usage()
1461 	printf("	[-3 loops] Number of loops for delay injection 3\n");  in show_usage()
1462 	printf("	[-4 loops] Number of loops for delay injection 4\n");  in show_usage()
1463 	printf("	[-5 loops] Number of loops for delay injection 5\n");  in show_usage()
1464 	printf("	[-6 loops] Number of loops for delay injection 6\n");  in show_usage()
1465 	printf("	[-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n");  in show_usage()
1466 	printf("	[-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n");  in show_usage()
1467 	printf("	[-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n");  in show_usage()
1468 	printf("	[-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n");  in show_usage()
1469 	printf("	[-y] Yield\n");  in show_usage()
1470 	printf("	[-k] Kill thread with signal\n");  in show_usage()
1471 	printf("	[-s S] S: =0: disabled (default), >0: sleep time (ms)\n");  in show_usage()
1472 	printf("	[-t N] Number of threads (default 200)\n");  in show_usage()
1473 	printf("	[-r N] Number of repetitions per thread (default 5000)\n");  in show_usage()
1474 	printf("	[-d] Disable rseq system call (no initialization)\n");  in show_usage()
1475 	printf("	[-D M] Disable rseq for each M threads\n");  in show_usage()
1476 …printf("	[-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement, membarrie(r)\…  in show_usage()
1477 	printf("	[-M] Push into buffer and memcpy buffer with memory barriers.\n");  in show_usage()
1478 	printf("	[-v] Verbose output.\n");  in show_usage()
1479 	printf("	[-h] Show this help.\n");  in show_usage()
1488 		if (argv[i][0] != '-')  in main()
1504 			loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]);  in main()
1624 		fprintf(stderr, "Error: cpu id getter unavailable\n");  in main()
1659 	return -1;  in main()