60 static void ringbuf_free_ring(struct ring_buffer *rb, struct ring *r)  in ringbuf_free_ring()  argument
63 		munmap(r->consumer_pos, rb->page_size);  in ringbuf_free_ring()
67 		munmap(r->producer_pos, rb->page_size + 2 * (r->mask + 1));  in ringbuf_free_ring()
75 int ring_buffer__add(struct ring_buffer *rb, int map_fd,  in ring_buffer__add()  argument
102 	tmp = libbpf_reallocarray(rb->rings, rb->ring_cnt + 1, sizeof(*rb->rings));  in ring_buffer__add()
105 	rb->rings = tmp;  in ring_buffer__add()
107 	tmp = libbpf_reallocarray(rb->events, rb->ring_cnt + 1, sizeof(*rb->events));  in ring_buffer__add()
110 	rb->events = tmp;  in ring_buffer__add()
115 	rb->rings[rb->ring_cnt] = r;  in ring_buffer__add()
123 	tmp = mmap(NULL, rb->page_size, PROT_READ | PROT_WRITE, MAP_SHARED, map_fd, 0);  in ring_buffer__add()
136 	mmap_sz = rb->page_size + 2 * (__u64)info.max_entries;  in ring_buffer__add()
142 	tmp = mmap(NULL, (size_t)mmap_sz, PROT_READ, MAP_SHARED, map_fd, rb->page_size);  in ring_buffer__add()
150 	r->data = tmp + rb->page_size;  in ring_buffer__add()
152 	e = &rb->events[rb->ring_cnt];  in ring_buffer__add()
156 	e->data.fd = rb->ring_cnt;  in ring_buffer__add()
157 	if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, e) < 0) {  in ring_buffer__add()
164 	rb->ring_cnt++;  in ring_buffer__add()
168 	ringbuf_free_ring(rb, r);  in ring_buffer__add()
172 void ring_buffer__free(struct ring_buffer *rb)  in ring_buffer__free()  argument
176 	if (!rb)  in ring_buffer__free()
179 	for (i = 0; i < rb->ring_cnt; ++i)  in ring_buffer__free()
180 		ringbuf_free_ring(rb, rb->rings[i]);  in ring_buffer__free()
181 	if (rb->epoll_fd >= 0)  in ring_buffer__free()
182 		close(rb->epoll_fd);  in ring_buffer__free()
184 	free(rb->events);  in ring_buffer__free()
185 	free(rb->rings);  in ring_buffer__free()
186 	free(rb);  in ring_buffer__free()
193 	struct ring_buffer *rb;  in ring_buffer__new()  local
199 	rb = calloc(1, sizeof(*rb));  in ring_buffer__new()
200 	if (!rb)  in ring_buffer__new()
203 	rb->page_size = getpagesize();  in ring_buffer__new()
205 	rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);  in ring_buffer__new()
206 	if (rb->epoll_fd < 0) {  in ring_buffer__new()
212 	err = ring_buffer__add(rb, map_fd, sample_cb, ctx);  in ring_buffer__new()
216 	return rb;  in ring_buffer__new()
219 	ring_buffer__free(rb);  in ring_buffer__new()
282 int ring_buffer__consume(struct ring_buffer *rb)  in ring_buffer__consume()  argument
287 	for (i = 0; i < rb->ring_cnt; i++) {  in ring_buffer__consume()
288 		struct ring *ring = rb->rings[i];  in ring_buffer__consume()
304 int ring_buffer__poll(struct ring_buffer *rb, int timeout_ms)  in ring_buffer__poll()  argument
309 	cnt = epoll_wait(rb->epoll_fd, rb->events, rb->ring_cnt, timeout_ms);  in ring_buffer__poll()
314 		__u32 ring_id = rb->events[i].data.fd;  in ring_buffer__poll()
315 		struct ring *ring = rb->rings[ring_id];  in ring_buffer__poll()
328 int ring_buffer__epoll_fd(const struct ring_buffer *rb)  in ring_buffer__epoll_fd()  argument
330 	return rb->epoll_fd;  in ring_buffer__epoll_fd()
333 struct ring *ring_buffer__ring(struct ring_buffer *rb, unsigned int idx)  in ring_buffer__ring()  argument
335 	if (idx >= rb->ring_cnt)  in ring_buffer__ring()
338 	return rb->rings[idx];  in ring_buffer__ring()
385 static void user_ringbuf_unmap_ring(struct user_ring_buffer *rb)  in user_ringbuf_unmap_ring()  argument
387 	if (rb->consumer_pos) {  in user_ringbuf_unmap_ring()
388 		munmap(rb->consumer_pos, rb->page_size);  in user_ringbuf_unmap_ring()
389 		rb->consumer_pos = NULL;  in user_ringbuf_unmap_ring()
391 	if (rb->producer_pos) {  in user_ringbuf_unmap_ring()
392 		munmap(rb->producer_pos, rb->page_size + 2 * (rb->mask + 1));  in user_ringbuf_unmap_ring()
393 		rb->producer_pos = NULL;  in user_ringbuf_unmap_ring()
397 void user_ring_buffer__free(struct user_ring_buffer *rb)  in user_ring_buffer__free()  argument
399 	if (!rb)  in user_ring_buffer__free()
402 	user_ringbuf_unmap_ring(rb);  in user_ring_buffer__free()
404 	if (rb->epoll_fd >= 0)  in user_ring_buffer__free()
405 		close(rb->epoll_fd);  in user_ring_buffer__free()
407 	free(rb);  in user_ring_buffer__free()
410 static int user_ringbuf_map(struct user_ring_buffer *rb, int map_fd)  in user_ringbuf_map()  argument
433 	rb->map_fd = map_fd;  in user_ringbuf_map()
434 	rb->mask = info.max_entries - 1;  in user_ringbuf_map()
437 	tmp = mmap(NULL, rb->page_size, PROT_READ, MAP_SHARED, map_fd, 0);  in user_ringbuf_map()
444 	rb->consumer_pos = tmp;  in user_ringbuf_map()
451 	mmap_sz = rb->page_size + 2 * (__u64)info.max_entries;  in user_ringbuf_map()
457 		   map_fd, rb->page_size);  in user_ringbuf_map()
465 	rb->producer_pos = tmp;  in user_ringbuf_map()
466 	rb->data = tmp + rb->page_size;  in user_ringbuf_map()
468 	rb_epoll = &rb->event;  in user_ringbuf_map()
470 	if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, rb_epoll) < 0) {  in user_ringbuf_map()
482 	struct user_ring_buffer *rb;  in user_ring_buffer__new()  local
488 	rb = calloc(1, sizeof(*rb));  in user_ring_buffer__new()
489 	if (!rb)  in user_ring_buffer__new()
492 	rb->page_size = getpagesize();  in user_ring_buffer__new()
494 	rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);  in user_ring_buffer__new()
495 	if (rb->epoll_fd < 0) {  in user_ring_buffer__new()
501 	err = user_ringbuf_map(rb, map_fd);  in user_ring_buffer__new()
505 	return rb;  in user_ring_buffer__new()
508 	user_ring_buffer__free(rb);  in user_ring_buffer__new()
512 static void user_ringbuf_commit(struct user_ring_buffer *rb, void *sample, bool discard)  in user_ringbuf_commit()  argument
518 	hdr_offset = rb->mask + 1 + (sample - rb->data) - BPF_RINGBUF_HDR_SZ;  in user_ringbuf_commit()
519 	hdr = rb->data + (hdr_offset & rb->mask);  in user_ringbuf_commit()
531 void user_ring_buffer__discard(struct user_ring_buffer *rb, void *sample)  in user_ring_buffer__discard()  argument
533 	user_ringbuf_commit(rb, sample, true);  in user_ring_buffer__discard()
536 void user_ring_buffer__submit(struct user_ring_buffer *rb, void *sample)  in user_ring_buffer__submit()  argument
538 	user_ringbuf_commit(rb, sample, false);  in user_ring_buffer__submit()
541 void *user_ring_buffer__reserve(struct user_ring_buffer *rb, __u32 size)  in user_ring_buffer__reserve()  argument
555 	cons_pos = smp_load_acquire(rb->consumer_pos);  in user_ring_buffer__reserve()
557 	prod_pos = smp_load_acquire(rb->producer_pos);  in user_ring_buffer__reserve()
559 	max_size = rb->mask + 1;  in user_ring_buffer__reserve()
570 	hdr = rb->data + (prod_pos & rb->mask);  in user_ring_buffer__reserve()
577 	smp_store_release(rb->producer_pos, prod_pos + total_size);  in user_ring_buffer__reserve()
579 	return (void *)rb->data + ((prod_pos + BPF_RINGBUF_HDR_SZ) & rb->mask);  in user_ring_buffer__reserve()
592 void *user_ring_buffer__reserve_blocking(struct user_ring_buffer *rb, __u32 size, int timeout_ms)  in user_ring_buffer__reserve_blocking()  argument
612 		sample = user_ring_buffer__reserve(rb, size);  in user_ring_buffer__reserve_blocking()
629 		cnt = epoll_wait(rb->epoll_fd, &rb->event, 1, ms_remaining);  in user_ring_buffer__reserve_blocking()
645 	return user_ring_buffer__reserve(rb, size);  in user_ring_buffer__reserve_blocking()