Lines Matching +full:cpu +full:- +full:offset
1 // SPDX-License-Identifier: LGPL-2.1
44 static __thread __attribute__((tls_model("initial-exec")))
49 static __thread __attribute__((tls_model("initial-exec"), unused))
113 "ahi %%" INJECT_ASM_REG ", -1\n\t" \
204 "addiu " INJECT_ASM_REG ", -1\n\t" \
226 "addi " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t" \
250 "l.addi " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t" \
269 if (loc_nr_loops == -1 && opt_modulo) { \
270 if (yield_mod_cnt == opt_modulo - 1) { \
326 int rseq_membarrier_expedited(int cpu) in rseq_membarrier_expedited() argument
351 int rseq_membarrier_expedited(int cpu) in rseq_membarrier_expedited() argument
354 MEMBARRIER_CMD_FLAG_CPU, cpu); in rseq_membarrier_expedited()
412 intptr_t offset; member
429 intptr_t offset; member
438 /* A simple percpu spinlock. Grabs lock on current cpu. */
441 int cpu; in rseq_this_cpu_lock() local
446 cpu = get_current_cpu_id(); in rseq_this_cpu_lock()
447 if (cpu < 0) { in rseq_this_cpu_lock()
448 fprintf(stderr, "pid: %d: tid: %d, cpu: %d: cid: %d\n", in rseq_this_cpu_lock()
449 getpid(), (int) rseq_gettid(), rseq_current_cpu_raw(), cpu); in rseq_this_cpu_lock()
453 &lock->c[cpu].v, in rseq_this_cpu_lock()
454 0, 1, cpu); in rseq_this_cpu_lock()
464 return cpu; in rseq_this_cpu_lock()
467 static void rseq_percpu_unlock(struct percpu_lock *lock, int cpu) in rseq_percpu_unlock() argument
469 assert(lock->c[cpu].v == 1); in rseq_percpu_unlock()
474 rseq_smp_store_release(&lock->c[cpu].v, 0); in rseq_percpu_unlock()
480 struct spinlock_test_data *data = thread_data->data; in test_percpu_spinlock_thread()
483 if (!opt_disable_rseq && thread_data->reg && in test_percpu_spinlock_thread()
486 reps = thread_data->reps; in test_percpu_spinlock_thread()
488 int cpu = rseq_this_cpu_lock(&data->lock); in test_percpu_spinlock_thread() local
489 data->c[cpu].count++; in test_percpu_spinlock_thread()
490 rseq_percpu_unlock(&data->lock, cpu); in test_percpu_spinlock_thread()
499 if (!opt_disable_rseq && thread_data->reg && in test_percpu_spinlock_thread()
506 * A simple test which implements a sharded counter using a per-cpu
508 * per-cpu increment; however, this is reasonable for a test and the
557 struct inc_test_data *data = thread_data->data; in test_percpu_inc_thread()
560 if (!opt_disable_rseq && thread_data->reg && in test_percpu_inc_thread()
563 reps = thread_data->reps; in test_percpu_inc_thread()
568 int cpu; in test_percpu_inc_thread() local
570 cpu = get_current_cpu_id(); in test_percpu_inc_thread()
572 &data->c[cpu].count, 1, cpu); in test_percpu_inc_thread()
582 if (!opt_disable_rseq && thread_data->reg && in test_percpu_inc_thread()
635 int cpu; in this_cpu_list_push() local
641 cpu = get_current_cpu_id(); in this_cpu_list_push()
642 /* Load list->c[cpu].head with single-copy atomicity. */ in this_cpu_list_push()
643 expect = (intptr_t)RSEQ_READ_ONCE(list->c[cpu].head); in this_cpu_list_push()
645 targetptr = (intptr_t *)&list->c[cpu].head; in this_cpu_list_push()
646 node->next = (struct percpu_list_node *)expect; in this_cpu_list_push()
648 targetptr, expect, newval, cpu); in this_cpu_list_push()
654 *_cpu = cpu; in this_cpu_list_push()
658 * Unlike a traditional lock-less linked list; the availability of a
660 * ABA-type races.
666 int cpu; in this_cpu_list_pop() local
671 long offset; in this_cpu_list_pop() local
674 cpu = get_current_cpu_id(); in this_cpu_list_pop()
675 targetptr = (intptr_t *)&list->c[cpu].head; in this_cpu_list_pop()
677 offset = offsetof(struct percpu_list_node, next); in this_cpu_list_pop()
681 offset, load, cpu); in this_cpu_list_pop()
691 *_cpu = cpu; in this_cpu_list_pop()
699 struct percpu_list_node *__percpu_list_pop(struct percpu_list *list, int cpu) in __percpu_list_pop() argument
703 node = list->c[cpu].head; in __percpu_list_pop()
706 list->c[cpu].head = node->next; in __percpu_list_pop()
737 /* Simultaneous modification to a per-cpu linked list from many threads. */
749 /* Generate list entries for every usable cpu. */ in test_percpu_list()
761 node->data = j; in test_percpu_list()
762 node->next = list.c[i].head; in test_percpu_list()
793 sum += node->data; in test_percpu_list()
811 int cpu; in this_cpu_buffer_push() local
816 intptr_t offset; in this_cpu_buffer_push() local
819 cpu = get_current_cpu_id(); in this_cpu_buffer_push()
820 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_buffer_push()
821 if (offset == buffer->c[cpu].buflen) in this_cpu_buffer_push()
824 targetptr_spec = (intptr_t *)&buffer->c[cpu].array[offset]; in this_cpu_buffer_push()
825 newval_final = offset + 1; in this_cpu_buffer_push()
826 targetptr_final = &buffer->c[cpu].offset; in this_cpu_buffer_push()
828 targetptr_final, offset, targetptr_spec, in this_cpu_buffer_push()
829 newval_spec, newval_final, cpu); in this_cpu_buffer_push()
837 *_cpu = cpu; in this_cpu_buffer_push()
845 int cpu; in this_cpu_buffer_pop() local
849 intptr_t offset; in this_cpu_buffer_pop() local
852 cpu = get_current_cpu_id(); in this_cpu_buffer_pop()
853 /* Load offset with single-copy atomicity. */ in this_cpu_buffer_pop()
854 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_buffer_pop()
855 if (offset == 0) { in this_cpu_buffer_pop()
859 head = RSEQ_READ_ONCE(buffer->c[cpu].array[offset - 1]); in this_cpu_buffer_pop()
860 newval = offset - 1; in this_cpu_buffer_pop()
861 targetptr = (intptr_t *)&buffer->c[cpu].offset; in this_cpu_buffer_pop()
863 targetptr, offset, in this_cpu_buffer_pop()
864 (intptr_t *)&buffer->c[cpu].array[offset - 1], in this_cpu_buffer_pop()
865 (intptr_t)head, newval, cpu); in this_cpu_buffer_pop()
871 *_cpu = cpu; in this_cpu_buffer_pop()
880 int cpu) in __percpu_buffer_pop() argument
883 intptr_t offset; in __percpu_buffer_pop() local
885 offset = buffer->c[cpu].offset; in __percpu_buffer_pop()
886 if (offset == 0) in __percpu_buffer_pop()
888 head = buffer->c[cpu].array[offset - 1]; in __percpu_buffer_pop()
889 buffer->c[cpu].offset = offset - 1; in __percpu_buffer_pop()
924 /* Simultaneous modification to a per-cpu buffer from many threads. */
936 /* Generate list entries for every usable cpu. */ in test_percpu_buffer()
941 /* Worse-case is every item in same CPU. */ in test_percpu_buffer()
953 * We could theoretically put the word-sized in test_percpu_buffer()
961 node->data = j; in test_percpu_buffer()
962 buffer.c[i].array[j - 1] = node; in test_percpu_buffer()
963 buffer.c[i].offset++; in test_percpu_buffer()
993 sum += node->data; in test_percpu_buffer()
1012 int cpu; in this_cpu_memcpy_buffer_push() local
1015 intptr_t *targetptr_final, newval_final, offset; in this_cpu_memcpy_buffer_push() local
1020 cpu = get_current_cpu_id(); in this_cpu_memcpy_buffer_push()
1021 /* Load offset with single-copy atomicity. */ in this_cpu_memcpy_buffer_push()
1022 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_memcpy_buffer_push()
1023 if (offset == buffer->c[cpu].buflen) in this_cpu_memcpy_buffer_push()
1025 destptr = (char *)&buffer->c[cpu].array[offset]; in this_cpu_memcpy_buffer_push()
1029 newval_final = offset + 1; in this_cpu_memcpy_buffer_push()
1030 targetptr_final = &buffer->c[cpu].offset; in this_cpu_memcpy_buffer_push()
1033 targetptr_final, offset, in this_cpu_memcpy_buffer_push()
1035 newval_final, cpu); in this_cpu_memcpy_buffer_push()
1043 *_cpu = cpu; in this_cpu_memcpy_buffer_push()
1052 int cpu; in this_cpu_memcpy_buffer_pop() local
1055 intptr_t *targetptr_final, newval_final, offset; in this_cpu_memcpy_buffer_pop() local
1060 cpu = get_current_cpu_id(); in this_cpu_memcpy_buffer_pop()
1061 /* Load offset with single-copy atomicity. */ in this_cpu_memcpy_buffer_pop()
1062 offset = RSEQ_READ_ONCE(buffer->c[cpu].offset); in this_cpu_memcpy_buffer_pop()
1063 if (offset == 0) in this_cpu_memcpy_buffer_pop()
1066 srcptr = (char *)&buffer->c[cpu].array[offset - 1]; in this_cpu_memcpy_buffer_pop()
1069 newval_final = offset - 1; in this_cpu_memcpy_buffer_pop()
1070 targetptr_final = &buffer->c[cpu].offset; in this_cpu_memcpy_buffer_pop()
1072 targetptr_final, offset, destptr, srcptr, copylen, in this_cpu_memcpy_buffer_pop()
1073 newval_final, cpu); in this_cpu_memcpy_buffer_pop()
1081 *_cpu = cpu; in this_cpu_memcpy_buffer_pop()
1091 int cpu) in __percpu_memcpy_buffer_pop() argument
1093 intptr_t offset; in __percpu_memcpy_buffer_pop() local
1095 offset = buffer->c[cpu].offset; in __percpu_memcpy_buffer_pop()
1096 if (offset == 0) in __percpu_memcpy_buffer_pop()
1098 memcpy(item, &buffer->c[cpu].array[offset - 1], sizeof(*item)); in __percpu_memcpy_buffer_pop()
1099 buffer->c[cpu].offset = offset - 1; in __percpu_memcpy_buffer_pop()
1135 /* Simultaneous modification to a per-cpu buffer from many threads. */
1147 /* Generate list entries for every usable cpu. */ in test_percpu_memcpy_buffer()
1152 /* Worse-case is every item in same CPU. */ in test_percpu_memcpy_buffer()
1162 * We could theoretically put the word-sized in test_percpu_memcpy_buffer()
1168 buffer.c[i].array[j - 1].data1 = j; in test_percpu_memcpy_buffer()
1169 buffer.c[i].array[j - 1].data2 = j + 1; in test_percpu_memcpy_buffer()
1170 buffer.c[i].offset++; in test_percpu_memcpy_buffer()
1268 while (!__atomic_load_n(&args->percpu_list_ptr, __ATOMIC_ACQUIRE)) {} in test_membarrier_worker_thread()
1274 int cpu = get_current_cpu_id(); in test_membarrier_worker_thread() local
1277 &args->percpu_list_ptr, in test_membarrier_worker_thread()
1278 sizeof(struct percpu_list_entry) * cpu, 1, cpu); in test_membarrier_worker_thread()
1300 node->data = 0; in test_membarrier_init_percpu_list()
1301 node->next = NULL; in test_membarrier_init_percpu_list()
1302 list->c[i].head = node; in test_membarrier_init_percpu_list()
1311 free(list->c[i].head); in test_membarrier_free_percpu_list()
1315 * The manager thread swaps per-cpu lists that worker threads see,
1336 __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE); in test_membarrier_manager_thread()
1338 while (!__atomic_load_n(&args->stop, __ATOMIC_ACQUIRE)) { in test_membarrier_manager_thread()
1345 if (expect_b != __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE)) { in test_membarrier_manager_thread()
1351 __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_b, __ATOMIC_RELEASE); in test_membarrier_manager_thread()
1353 errno != ENXIO /* missing CPU */) { in test_membarrier_manager_thread()
1358 * Cpu A should now only modify list_b, so the values in test_membarrier_manager_thread()
1361 expect_a = __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE); in test_membarrier_manager_thread()
1368 if (expect_a != __atomic_load_n(&list_a.c[cpu_a].head->data, __ATOMIC_ACQUIRE)) { in test_membarrier_manager_thread()
1374 __atomic_store_n(&args->percpu_list_ptr, (intptr_t)&list_a, __ATOMIC_RELEASE); in test_membarrier_manager_thread()
1376 errno != ENXIO /* missing CPU*/) { in test_membarrier_manager_thread()
1381 expect_b = __atomic_load_n(&list_b.c[cpu_b].head->data, __ATOMIC_ACQUIRE); in test_membarrier_manager_thread()
1459 printf(" [-1 loops] Number of loops for delay injection 1\n"); in show_usage()
1460 printf(" [-2 loops] Number of loops for delay injection 2\n"); in show_usage()
1461 printf(" [-3 loops] Number of loops for delay injection 3\n"); in show_usage()
1462 printf(" [-4 loops] Number of loops for delay injection 4\n"); in show_usage()
1463 printf(" [-5 loops] Number of loops for delay injection 5\n"); in show_usage()
1464 printf(" [-6 loops] Number of loops for delay injection 6\n"); in show_usage()
1465 printf(" [-7 loops] Number of loops for delay injection 7 (-1 to enable -m)\n"); in show_usage()
1466 printf(" [-8 loops] Number of loops for delay injection 8 (-1 to enable -m)\n"); in show_usage()
1467 printf(" [-9 loops] Number of loops for delay injection 9 (-1 to enable -m)\n"); in show_usage()
1468 printf(" [-m N] Yield/sleep/kill every modulo N (default 0: disabled) (>= 0)\n"); in show_usage()
1469 printf(" [-y] Yield\n"); in show_usage()
1470 printf(" [-k] Kill thread with signal\n"); in show_usage()
1471 printf(" [-s S] S: =0: disabled (default), >0: sleep time (ms)\n"); in show_usage()
1472 printf(" [-t N] Number of threads (default 200)\n"); in show_usage()
1473 printf(" [-r N] Number of repetitions per thread (default 5000)\n"); in show_usage()
1474 printf(" [-d] Disable rseq system call (no initialization)\n"); in show_usage()
1475 printf(" [-D M] Disable rseq for each M threads\n"); in show_usage()
1476 …printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement, membarrie(r)\… in show_usage()
1477 printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n"); in show_usage()
1478 printf(" [-v] Verbose output.\n"); in show_usage()
1479 printf(" [-h] Show this help.\n"); in show_usage()
1488 if (argv[i][0] != '-') in main()
1504 loop_cnt[argv[i][1] - '0'] = atol(argv[i + 1]); in main()
1624 fprintf(stderr, "Error: cpu id getter unavailable\n"); in main()
1659 return -1; in main()