xref: /aosp_15_r20/external/ublksrv/include/ublksrv_priv.h (revision 94c4a1e103eb1715230460aab379dff275992c20)

// SPDX-License-Identifier: MIT or GPL-2.0-only

#if !defined(UBLKSRV_INTERNAL_H_)
#error "Never include <ublksrv_priv.h> directly; use <ublksrv.h> instead."
#endif

#ifndef UBLKSRV_PRIVATE_INC_H
#define UBLKSRV_PRIVATE_INC_H

#include <unistd.h>
#include <stdlib.h>
#include <stddef.h>
#include <signal.h>
#include <limits.h>
#include <pthread.h>
#include <string.h>
#include <sys/types.h>
#include <sys/eventfd.h>
#include <sys/epoll.h>
#include <sys/poll.h>

#include "ublk_cmd.h"
#include "ublksrv_utils.h"
#include "ublksrv.h"
#include "ublksrv_aio.h"


/* todo: relace the hardcode name with /dev/char/maj:min */
#define UBLKC_DEV	"/dev/ublkc"
#define UBLKC_PATH_MAX	32

#ifdef __cplusplus
extern "C" {
#endif

struct ublksrv_ctrl_dev {
	struct io_uring ring;

	int ctrl_fd;
	unsigned bs_shift;
	struct ublksrv_ctrl_dev_info  dev_info;

	const char *tgt_type;
	const struct ublksrv_tgt_type *tgt_ops;

	/*
	 * default is UBLKSRV_RUN_DIR but can be specified via command line,
	 * pid file will be saved there
	 */
	const char *run_dir;

	union {
		/* used by ->init_tgt() */
		struct {
			int tgt_argc;
			char **tgt_argv;
		};
		/* used by ->recovery_tgt(), tgt_argc == -1 */
		struct {
			int padding;
			const char *recovery_jbuf;
		};
	};

	cpu_set_t *queues_cpuset;

	unsigned long reserved[4];
};

struct ublk_io {
	char *buf_addr;

#define UBLKSRV_NEED_FETCH_RQ		(1UL << 0)
#define UBLKSRV_NEED_COMMIT_RQ_COMP	(1UL << 1)
#define UBLKSRV_IO_FREE			(1UL << 2)
#define UBLKSRV_NEED_GET_DATA		(1UL << 3)
	unsigned int flags;

	/* result is updated after all target ios are done */
	unsigned int result;

	struct ublk_io_data  data;
};

struct _ublksrv_queue {
	/********** part of API, can't change ************/
	int q_id;
	int q_depth;

	struct io_uring *ring_ptr;
	struct _ublksrv_dev *dev;
	void *private_data;
	/*************************************************/

	/*
	 * Read only by ublksrv daemon, setup via mmap on /dev/ublkcN.
	 *
	 * ublksrv_io_desc(iod) is stored in this buffer, so iod
	 * can be retrieved by request's tag directly.
	 *
	 * ublksrv writes the iod into this array, and notify ublksrv daemon
	 * by issued io_uring command beforehand.
	 * */
	char *io_cmd_buf;
	char *io_buf;

	unsigned cmd_inflight, tgt_io_inflight;	//obsolete
	unsigned state;

	/* eventfd */
	int efd;

	/* cache tgt ops */
	const struct ublksrv_tgt_type *tgt_ops;

	/*
	 * ring for submit io command to ublk driver, can only be issued
	 * from ublksrv daemon.
	 *
	 * ring depth == dev_info->queue_depth.
	 */
	struct io_uring ring;

	unsigned  tid;

#define UBLKSRV_NR_CTX_BATCH 4
	int nr_ctxs;
	struct ublksrv_aio_ctx *ctxs[UBLKSRV_NR_CTX_BATCH];

	unsigned long reserved[8];

	struct ublk_io ios[0];
};

struct _ublksrv_dev {
	//keep same with ublksrv_dev
	/********** part of API, can't change ************/
	struct ublksrv_tgt_info tgt;
	/************************************************/

	struct _ublksrv_queue *__queues[MAX_NR_HW_QUEUES];
	char	*io_buf_start;
	pthread_t *thread;
	int cdev_fd;
	int pid_file_fd;

	const struct ublksrv_ctrl_dev *ctrl_dev;
	void	*target_data;
	int	cq_depth;
	int	pad;

	/* reserved isn't necessary any more */
	unsigned long reserved[3];
};

#define local_to_tq(q)	((struct ublksrv_queue *)(q))
#define tq_to_local(q)	((struct _ublksrv_queue *)(q))

#define local_to_tdev(d)	((struct ublksrv_dev *)(d))
#define tdev_to_local(d)	((struct _ublksrv_dev *)(d))

static inline bool ublk_is_unprivileged(const struct ublksrv_ctrl_dev *ctrl_dev)
{
	return !!(ctrl_dev->dev_info.flags & UBLK_F_UNPRIVILEGED_DEV);
}

static inline cpu_set_t *ublksrv_get_queue_affinity(
		const struct ublksrv_ctrl_dev *dev, int qid)
{
	unsigned char *buf = (unsigned char *)&dev->queues_cpuset[qid];

	if (ublk_is_unprivileged(dev))
		return (cpu_set_t *)&buf[UBLKC_PATH_MAX];

	return &dev->queues_cpuset[qid];
}

static inline void ublksrv_mark_io_done(struct ublk_io *io, int res)
{
	/*
	 * mark io done by target, so that ->ubq_daemon can commit its
	 * result and fetch new request via io_uring command.
	 */
	io->flags |= (UBLKSRV_NEED_COMMIT_RQ_COMP | UBLKSRV_IO_FREE);

	io->result = res;
}

static inline bool ublksrv_io_done(struct ublk_io *io)
{
	return io->flags & UBLKSRV_IO_FREE;
}

int create_pid_file(const char *pid_file, int *pid_fd);

extern void ublksrv_build_cpu_str(char *buf, int len, const cpu_set_t *cpuset);

/* bit63: target io, bit62: eventfd data */
static inline __u64 build_eventfd_data()
{
	return 0x3ULL << 62;
}

static inline int is_eventfd_io(__u64 user_data)
{
	return (user_data & (1ULL << 62)) != 0;
}

static inline int is_target_io(__u64 user_data)
{
	return (user_data & (1ULL << 63)) != 0;
}

/* two helpers for setting up io_uring */
static inline int ublksrv_setup_ring(struct io_uring *r, int depth,
		int cq_depth, unsigned flags)
{
	struct io_uring_params p;

	memset(&p, 0, sizeof(p));
	p.flags = flags | IORING_SETUP_CQSIZE;
	p.cq_entries = cq_depth;

	return io_uring_queue_init_params(depth, r, &p);
}

static inline struct io_uring_sqe *ublksrv_uring_get_sqe(struct io_uring *r,
		int idx, bool is_sqe128)
{
	if (is_sqe128)
		return  &r->sq.sqes[idx << 1];
	return  &r->sq.sqes[idx];
}

static inline void *ublksrv_get_sqe_cmd(struct io_uring_sqe *sqe)
{
	return (void *)&sqe->addr3;
}

static inline void ublksrv_set_sqe_cmd_op(struct io_uring_sqe *sqe, __u32 cmd_op)
{
	__u32 *addr = (__u32 *)&sqe->off;

	addr[0] = cmd_op;
	addr[1] = 0;
}

/*
 * ublksrv_aio_ctx is used to offload IO handling from ublksrv io_uring
 * context.
 *
 * ublksrv_aio_ctx is bound with one single pthread which has to belong
 * to same process of the io_uring where IO is originated, so we can
 * support to handle IO from multiple queues of the same device. At
 * default, ublksrv_aio_ctx supports to handle device wide aio or io
 * offloading except for UBLKSRV_AIO_QUEUE_WIDE.
 *
 * Meantime ublksrv_aio_ctx can be created per each queue, and only handle
 * IOs from this queue.
 *
 * The final io handling in the aio context depends on user's implementation,
 * either sync or async IO submitting is supported.
 */
struct ublksrv_aio_ctx {
	struct ublksrv_aio_list submit;

	/* per-queue completion list */
	struct ublksrv_aio_list *complete;

	int efd;		//for wakeup us

#define UBLKSRV_AIO_QUEUE_WIDE	(1U << 0)
	unsigned int		flags;
	bool dead;

	const struct ublksrv_dev *dev;

	void *ctx_data;

	unsigned long reserved[8];
};

#ifdef __cplusplus
}
#endif

#endif