1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3  * A generic kernel FIFO implementation
12  * How to porting drivers to the new generic FIFO API:
14  * - Modify the declaration of the "struct kfifo *" object into a
15  *   in-place "struct kfifo" object
16  * - Init the in-place object with kfifo_alloc() or kfifo_init()
17  *   Note: The address of the in-place "struct kfifo" object must be
19  * - Replace the use of __kfifo_put into kfifo_in and __kfifo_get
21  * - Replace the use of kfifo_put into kfifo_in_spinlocked and kfifo_get
26  * - The formerly __kfifo_* functions are renamed into kfifo_*
31  * and one writer is using the fifo and no kfifo_reset() will be called.
67 #define __STRUCT_KFIFO(type, size, recsize, ptrtype) \  argument
70 	type		buf[((size < 2) || (size & (size - 1))) ? -1 : size]; \
73 #define STRUCT_KFIFO(type, size) \  argument
74 	struct __STRUCT_KFIFO(type, size, 0, type)
90 #define STRUCT_KFIFO_REC_1(size) \  argument
91 	struct __STRUCT_KFIFO(unsigned char, size, 1, void)
93 #define STRUCT_KFIFO_REC_2(size) \  argument
94 	struct __STRUCT_KFIFO(unsigned char, size, 2, void)
103  * helper macro to distinguish between real in place fifo where the fifo
104  * array is a part of the structure and the fifo type where the array is
105  * outside of the fifo structure.
107 #define	__is_kfifo_ptr(fifo) \  argument
108 	(sizeof(*fifo) == sizeof(STRUCT_KFIFO_PTR(typeof(*(fifo)->type))))
111  * DECLARE_KFIFO_PTR - macro to declare a fifo pointer object
112  * @fifo: name of the declared fifo
113  * @type: type of the fifo elements
115 #define DECLARE_KFIFO_PTR(fifo, type)	STRUCT_KFIFO_PTR(type) fifo  argument
118  * DECLARE_KFIFO - macro to declare a fifo object
119  * @fifo: name of the declared fifo
120  * @type: type of the fifo elements
121  * @size: the number of elements in the fifo, this must be a power of 2
123 #define DECLARE_KFIFO(fifo, type, size)	STRUCT_KFIFO(type, size) fifo  argument
126  * INIT_KFIFO - Initialize a fifo declared by DECLARE_KFIFO
127  * @fifo: name of the declared fifo datatype
129 #define INIT_KFIFO(fifo) \  argument
131 	typeof(&(fifo)) __tmp = &(fifo); \
132 	struct __kfifo *__kfifo = &__tmp->kfifo; \
133 	__kfifo->in = 0; \
134 	__kfifo->out = 0; \
135 	__kfifo->mask = __is_kfifo_ptr(__tmp) ? 0 : ARRAY_SIZE(__tmp->buf) - 1;\
136 	__kfifo->esize = sizeof(*__tmp->buf); \
137 	__kfifo->data = __is_kfifo_ptr(__tmp) ?  NULL : __tmp->buf; \
141  * DEFINE_KFIFO - macro to define and initialize a fifo
142  * @fifo: name of the declared fifo datatype
143  * @type: type of the fifo elements
144  * @size: the number of elements in the fifo, this must be a power of 2
146  * Note: the macro can be used for global and local fifo data type variables.
148 #define DEFINE_KFIFO(fifo, type, size) \  argument
149 	DECLARE_KFIFO(fifo, type, size) = \
150 	(typeof(fifo)) { \
155 			.mask	= __is_kfifo_ptr(&(fifo)) ? \
157 				  ARRAY_SIZE((fifo).buf) - 1, \
158 			.esize	= sizeof(*(fifo).buf), \
159 			.data	= __is_kfifo_ptr(&(fifo)) ? \
161 				(fifo).buf, \
180  * kfifo_initialized - Check if the fifo is initialized
181  * @fifo: address of the fifo to check
183  * Return %true if fifo is initialized, otherwise %false.
184  * Assumes the fifo was 0 before.
186 #define kfifo_initialized(fifo) ((fifo)->kfifo.mask)  argument
189  * kfifo_esize - returns the size of the element managed by the fifo
190  * @fifo: address of the fifo to be used
192 #define kfifo_esize(fifo)	((fifo)->kfifo.esize)  argument
195  * kfifo_recsize - returns the size of the record length field
196  * @fifo: address of the fifo to be used
198 #define kfifo_recsize(fifo)	(sizeof(*(fifo)->rectype))  argument
201  * kfifo_size - returns the size of the fifo in elements
202  * @fifo: address of the fifo to be used
204 #define kfifo_size(fifo)	((fifo)->kfifo.mask + 1)  argument
207  * kfifo_reset - removes the entire fifo content
208  * @fifo: address of the fifo to be used
211  * fifo is exclusived locked or when it is secured that no other thread is
212  * accessing the fifo.
214 #define kfifo_reset(fifo) \  argument
216 	typeof((fifo) + 1) __tmp = (fifo); \
217 	__tmp->kfifo.in = __tmp->kfifo.out = 0; \
221  * kfifo_reset_out - skip fifo content
222  * @fifo: address of the fifo to be used
228 #define kfifo_reset_out(fifo)	\  argument
230 	typeof((fifo) + 1) __tmp = (fifo); \
231 	__tmp->kfifo.out = __tmp->kfifo.in; \
235  * kfifo_len - returns the number of used elements in the fifo
236  * @fifo: address of the fifo to be used
238 #define kfifo_len(fifo) \  argument
240 	typeof((fifo) + 1) __tmpl = (fifo); \
241 	__tmpl->kfifo.in - __tmpl->kfifo.out; \
245  * kfifo_is_empty - returns true if the fifo is empty
246  * @fifo: address of the fifo to be used
248 #define	kfifo_is_empty(fifo) \  argument
250 	typeof((fifo) + 1) __tmpq = (fifo); \
251 	__tmpq->kfifo.in == __tmpq->kfifo.out; \
255  * kfifo_is_empty_spinlocked - returns true if the fifo is empty using
257  * @fifo: address of the fifo to be used
260 #define kfifo_is_empty_spinlocked(fifo, lock) \  argument
265 	__ret = kfifo_is_empty(fifo); \
271  * kfifo_is_empty_spinlocked_noirqsave  - returns true if the fifo is empty
273  * @fifo: address of the fifo to be used
276 #define kfifo_is_empty_spinlocked_noirqsave(fifo, lock) \  argument
280 	__ret = kfifo_is_empty(fifo); \
286  * kfifo_is_full - returns true if the fifo is full
287  * @fifo: address of the fifo to be used
289 #define	kfifo_is_full(fifo) \  argument
291 	typeof((fifo) + 1) __tmpq = (fifo); \
292 	kfifo_len(__tmpq) > __tmpq->kfifo.mask; \
296  * kfifo_avail - returns the number of unused elements in the fifo
297  * @fifo: address of the fifo to be used
299 #define	kfifo_avail(fifo) \  argument
302 	typeof((fifo) + 1) __tmpq = (fifo); \
303 	const size_t __recsize = sizeof(*__tmpq->rectype); \
304 	unsigned int __avail = kfifo_size(__tmpq) - kfifo_len(__tmpq); \
306 	__kfifo_max_r(__avail - __recsize, __recsize)) : \
312  * kfifo_skip_count - skip output data
313  * @fifo: address of the fifo to be used
316 #define	kfifo_skip_count(fifo, count) do { \  argument
317 	typeof((fifo) + 1) __tmp = (fifo); \
318 	const size_t __recsize = sizeof(*__tmp->rectype); \
319 	struct __kfifo *__kfifo = &__tmp->kfifo; \
323 		__kfifo->out += (count); \
327  * kfifo_skip - skip output data
328  * @fifo: address of the fifo to be used
330 #define	kfifo_skip(fifo)	kfifo_skip_count(fifo, 1)  argument
333  * kfifo_peek_len - gets the size of the next fifo record
334  * @fifo: address of the fifo to be used
336  * This function returns the size of the next fifo record in number of bytes.
338 #define kfifo_peek_len(fifo) \  argument
341 	typeof((fifo) + 1) __tmp = (fifo); \
342 	const size_t __recsize = sizeof(*__tmp->rectype); \
343 	struct __kfifo *__kfifo = &__tmp->kfifo; \
344 	(!__recsize) ? kfifo_len(__tmp) * sizeof(*__tmp->type) : \
350  * kfifo_alloc - dynamically allocates a new fifo buffer
351  * @fifo: pointer to the fifo
352  * @size: the number of elements in the fifo, this must be a power of 2
355  * This macro dynamically allocates a new fifo buffer.
357  * The number of elements will be rounded-up to a power of 2.
358  * The fifo will be release with kfifo_free().
361 #define kfifo_alloc(fifo, size, gfp_mask) \  argument
364 	typeof((fifo) + 1) __tmp = (fifo); \
365 	struct __kfifo *__kfifo = &__tmp->kfifo; \
367 	__kfifo_alloc(__kfifo, size, sizeof(*__tmp->type), gfp_mask) : \
368 	-EINVAL; \
373  * kfifo_free - frees the fifo
374  * @fifo: the fifo to be freed
376 #define kfifo_free(fifo) \  argument
378 	typeof((fifo) + 1) __tmp = (fifo); \
379 	struct __kfifo *__kfifo = &__tmp->kfifo; \
385  * kfifo_init - initialize a fifo using a preallocated buffer
386  * @fifo: the fifo to assign the buffer
388  * @size: the size of the internal buffer, this have to be a power of 2
390  * This macro initializes a fifo using a preallocated buffer.
392  * The number of elements will be rounded-up to a power of 2.
395 #define kfifo_init(fifo, buffer, size) \  argument
397 	typeof((fifo) + 1) __tmp = (fifo); \
398 	struct __kfifo *__kfifo = &__tmp->kfifo; \
400 	__kfifo_init(__kfifo, buffer, size, sizeof(*__tmp->type)) : \
401 	-EINVAL; \
405  * kfifo_put - put data into the fifo
406  * @fifo: address of the fifo to be used
409  * This macro copies the given value into the fifo.
410  * It returns 0 if the fifo was full. Otherwise it returns the number
416 #define	kfifo_put(fifo, val) \  argument
418 	typeof((fifo) + 1) __tmp = (fifo); \
419 	typeof(*__tmp->const_type) __val = (val); \
421 	size_t __recsize = sizeof(*__tmp->rectype); \
422 	struct __kfifo *__kfifo = &__tmp->kfifo; \
430 			((typeof(__tmp->type))__kfifo->data) : \
431 			(__tmp->buf) \
432 			)[__kfifo->in & __tmp->kfifo.mask] = \
433 				*(typeof(__tmp->type))&__val; \
435 			__kfifo->in++; \
442  * kfifo_get - get data from the fifo
443  * @fifo: address of the fifo to be used
446  * This macro reads the data from the fifo.
447  * It returns 0 if the fifo was empty. Otherwise it returns the number
453 #define	kfifo_get(fifo, val) \  argument
456 	typeof((fifo) + 1) __tmp = (fifo); \
457 	typeof(__tmp->ptr) __val = (val); \
459 	const size_t __recsize = sizeof(*__tmp->rectype); \
460 	struct __kfifo *__kfifo = &__tmp->kfifo; \
467 			*(typeof(__tmp->type))__val = \
469 				((typeof(__tmp->type))__kfifo->data) : \
470 				(__tmp->buf) \
471 				)[__kfifo->out & __tmp->kfifo.mask]; \
473 			__kfifo->out++; \
481  * kfifo_peek - get data from the fifo without removing
482  * @fifo: address of the fifo to be used
485  * This reads the data from the fifo without removing it from the fifo.
486  * It returns 0 if the fifo was empty. Otherwise it returns the number
492 #define	kfifo_peek(fifo, val) \  argument
495 	typeof((fifo) + 1) __tmp = (fifo); \
496 	typeof(__tmp->ptr) __val = (val); \
498 	const size_t __recsize = sizeof(*__tmp->rectype); \
499 	struct __kfifo *__kfifo = &__tmp->kfifo; \
506 			*(typeof(__tmp->type))__val = \
508 				((typeof(__tmp->type))__kfifo->data) : \
509 				(__tmp->buf) \
510 				)[__kfifo->out & __tmp->kfifo.mask]; \
519  * kfifo_in - put data into the fifo
520  * @fifo: address of the fifo to be used
524  * This macro copies the given buffer into the fifo and returns the
530 #define	kfifo_in(fifo, buf, n) \  argument
532 	typeof((fifo) + 1) __tmp = (fifo); \
533 	typeof(__tmp->ptr_const) __buf = (buf); \
535 	const size_t __recsize = sizeof(*__tmp->rectype); \
536 	struct __kfifo *__kfifo = &__tmp->kfifo; \
543  * kfifo_in_spinlocked - put data into the fifo using a spinlock for locking
544  * @fifo: address of the fifo to be used
549  * This macro copies the given values buffer into the fifo and returns the
552 #define	kfifo_in_spinlocked(fifo, buf, n, lock) \  argument
557 	__ret = kfifo_in(fifo, buf, n); \
563  * kfifo_in_spinlocked_noirqsave - put data into fifo using a spinlock for
565  * @fifo: address of the fifo to be used
573 #define kfifo_in_spinlocked_noirqsave(fifo, buf, n, lock) \  argument
577 	__ret = kfifo_in(fifo, buf, n); \
583 #define kfifo_in_locked(fifo, buf, n, lock) \  argument
584 		kfifo_in_spinlocked(fifo, buf, n, lock)
587  * kfifo_out - get data from the fifo
588  * @fifo: address of the fifo to be used
592  * This macro gets some data from the fifo and returns the numbers of elements
598 #define	kfifo_out(fifo, buf, n) \  argument
601 	typeof((fifo) + 1) __tmp = (fifo); \
602 	typeof(__tmp->ptr) __buf = (buf); \
604 	const size_t __recsize = sizeof(*__tmp->rectype); \
605 	struct __kfifo *__kfifo = &__tmp->kfifo; \
613  * kfifo_out_spinlocked - get data from the fifo using a spinlock for locking
614  * @fifo: address of the fifo to be used
619  * This macro gets the data from the fifo and returns the numbers of elements
622 #define	kfifo_out_spinlocked(fifo, buf, n, lock) \  argument
628 	__ret = kfifo_out(fifo, buf, n); \
635  * kfifo_out_spinlocked_noirqsave - get data from the fifo using a spinlock
637  * @fifo: address of the fifo to be used
645 #define kfifo_out_spinlocked_noirqsave(fifo, buf, n, lock) \  argument
650 	__ret = kfifo_out(fifo, buf, n); \
657 #define kfifo_out_locked(fifo, buf, n, lock) \  argument
658 		kfifo_out_spinlocked(fifo, buf, n, lock)
661  * kfifo_from_user - puts some data from user space into the fifo
662  * @fifo: address of the fifo to be used
668  * fifo, depending of the available space and returns -EFAULT/0.
673 #define	kfifo_from_user(fifo, from, len, copied) \  argument
676 	typeof((fifo) + 1) __tmp = (fifo); \
680 	const size_t __recsize = sizeof(*__tmp->rectype); \
681 	struct __kfifo *__kfifo = &__tmp->kfifo; \
689  * kfifo_to_user - copies data from the fifo into user space
690  * @fifo: address of the fifo to be used
692  * @len: the size of the destination buffer
695  * This macro copies at most @len bytes from the fifo into the
696  * @to buffer and returns -EFAULT/0.
701 #define	kfifo_to_user(fifo, to, len, copied) \  argument
704 	typeof((fifo) + 1) __tmp = (fifo); \
708 	const size_t __recsize = sizeof(*__tmp->rectype); \
709 	struct __kfifo *__kfifo = &__tmp->kfifo; \
717  * kfifo_dma_in_prepare_mapped - setup a scatterlist for DMA input
718  * @fifo: address of the fifo to be used
730 #define	kfifo_dma_in_prepare_mapped(fifo, sgl, nents, len, dma) \  argument
732 	typeof((fifo) + 1) __tmp = (fifo); \
736 	const size_t __recsize = sizeof(*__tmp->rectype); \
737 	struct __kfifo *__kfifo = &__tmp->kfifo; \
744 #define kfifo_dma_in_prepare(fifo, sgl, nents, len) \  argument
745 	kfifo_dma_in_prepare_mapped(fifo, sgl, nents, len, DMA_MAPPING_ERROR)
748  * kfifo_dma_in_finish - finish a DMA IN operation
749  * @fifo: address of the fifo to be used
758 #define kfifo_dma_in_finish(fifo, len) \  argument
760 	typeof((fifo) + 1) __tmp = (fifo); \
762 	const size_t __recsize = sizeof(*__tmp->rectype); \
763 	struct __kfifo *__kfifo = &__tmp->kfifo; \
767 		__kfifo->in += __len / sizeof(*__tmp->type); \
771  * kfifo_dma_out_prepare_mapped - setup a scatterlist for DMA output
772  * @fifo: address of the fifo to be used
786 #define	kfifo_dma_out_prepare_mapped(fifo, sgl, nents, len, dma) \  argument
788 	typeof((fifo) + 1) __tmp = (fifo);  \
792 	const size_t __recsize = sizeof(*__tmp->rectype); \
793 	struct __kfifo *__kfifo = &__tmp->kfifo; \
800 #define	kfifo_dma_out_prepare(fifo, sgl, nents, len) \  argument
801 	kfifo_dma_out_prepare_mapped(fifo, sgl, nents, len, DMA_MAPPING_ERROR)
804  * kfifo_dma_out_finish - finish a DMA OUT operation
805  * @fifo: address of the fifo to be used
814 #define kfifo_dma_out_finish(fifo, len) do { \  argument
815 	typeof((fifo) + 1) ___tmp = (fifo); \
816 	kfifo_skip_count(___tmp, (len) / sizeof(*___tmp->type)); \
820  * kfifo_out_peek - gets some data from the fifo
821  * @fifo: address of the fifo to be used
825  * This macro gets the data from the fifo and returns the numbers of elements
826  * copied. The data is not removed from the fifo.
831 #define	kfifo_out_peek(fifo, buf, n) \  argument
834 	typeof((fifo) + 1) __tmp = (fifo); \
835 	typeof(__tmp->ptr) __buf = (buf); \
837 	const size_t __recsize = sizeof(*__tmp->rectype); \
838 	struct __kfifo *__kfifo = &__tmp->kfifo; \
846  * kfifo_out_linear - gets a tail of/offset to available data
847  * @fifo: address of the fifo to be used
851  * This macro obtains the offset (tail) to the available data in the fifo
855  * data processing (like memcpy() of (@fifo->data + @tail) with count
861 #define kfifo_out_linear(fifo, tail, n) \  argument
864 	typeof((fifo) + 1) __tmp = (fifo); \
867 	const size_t __recsize = sizeof(*__tmp->rectype); \
868 	struct __kfifo *__kfifo = &__tmp->kfifo; \
876  * kfifo_out_linear_ptr - gets a pointer to the available data
877  * @fifo: address of the fifo to be used
882  * available data in the fifo buffer and returns the numbers of elements
890 #define kfifo_out_linear_ptr(fifo, ptr, n) \  argument
893 	typeof((fifo) + 1) ___tmp = (fifo); \
896 	*(ptr) = ___tmp->kfifo.data + ___tail * kfifo_esize(___tmp); \
902 extern int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
905 extern void __kfifo_free(struct __kfifo *fifo);
907 extern int __kfifo_init(struct __kfifo *fifo, void *buffer,
908 	unsigned int size, size_t esize);
910 extern unsigned int __kfifo_in(struct __kfifo *fifo,
913 extern unsigned int __kfifo_out(struct __kfifo *fifo,
916 extern int __kfifo_from_user(struct __kfifo *fifo,
919 extern int __kfifo_to_user(struct __kfifo *fifo,
922 extern unsigned int __kfifo_dma_in_prepare(struct __kfifo *fifo,
925 extern unsigned int __kfifo_dma_out_prepare(struct __kfifo *fifo,
928 extern unsigned int __kfifo_out_peek(struct __kfifo *fifo,
931 extern unsigned int __kfifo_out_linear(struct __kfifo *fifo,
934 extern unsigned int __kfifo_in_r(struct __kfifo *fifo,
937 extern unsigned int __kfifo_out_r(struct __kfifo *fifo,
940 extern int __kfifo_from_user_r(struct __kfifo *fifo,
944 extern int __kfifo_to_user_r(struct __kfifo *fifo, void __user *to,
947 extern unsigned int __kfifo_dma_in_prepare_r(struct __kfifo *fifo,
951 extern void __kfifo_dma_in_finish_r(struct __kfifo *fifo,
954 extern unsigned int __kfifo_dma_out_prepare_r(struct __kfifo *fifo,
958 extern unsigned int __kfifo_len_r(struct __kfifo *fifo, size_t recsize);
960 extern void __kfifo_skip_r(struct __kfifo *fifo, size_t recsize);
962 extern unsigned int __kfifo_out_peek_r(struct __kfifo *fifo,
965 extern unsigned int __kfifo_out_linear_r(struct __kfifo *fifo,