diff options
| -rw-r--r-- | lib/fuse_lowlevel.c | 70 | ||||
| -rw-r--r-- | lib/fuse_uring.c | 150 | ||||
| -rw-r--r-- | lib/fuse_uring_i.h | 29 |
3 files changed, 222 insertions, 27 deletions
diff --git a/lib/fuse_lowlevel.c b/lib/fuse_lowlevel.c index d6b254e..03aeca7 100644 --- a/lib/fuse_lowlevel.c +++ b/lib/fuse_lowlevel.c @@ -227,21 +227,22 @@ static struct fuse_req *fuse_ll_alloc_req(struct fuse_session *se) /* * Send data to fuse-kernel using an fd of the fuse device. */ -static int _fuse_send_msg(struct fuse_session *se, struct fuse_chan *ch, - struct iovec *iov, int count) +static int fuse_write_msg_dev(struct fuse_session *se, struct fuse_chan *ch, + struct iovec *iov, int count) { ssize_t res; int err; - if (se->io != NULL) { + if (se->io != NULL) + /* se->io->writev is never NULL if se->io is not NULL as * specified by fuse_session_custom_io() */ res = se->io->writev(ch ? ch->fd : se->fd, iov, count, se->userdata); - } else { + else res = writev(ch ? ch->fd : se->fd, iov, count); - } + if (res == -1) { /* ENOENT means the operation was interrupted */ err = errno; @@ -253,15 +254,17 @@ static int _fuse_send_msg(struct fuse_session *se, struct fuse_chan *ch, return 0; } -/* Send data. If *ch* is NULL, send via session master fd */ static int fuse_send_msg(struct fuse_session *se, struct fuse_chan *ch, - struct iovec *iov, int count) + struct iovec *iov, int count, fuse_req_t req) { struct fuse_out_header *out = iov[0].iov_base; int err; + bool is_uring = req && req->is_uring ? true : false; - assert(se != NULL); + if (!is_uring) + assert(se != NULL); out->len = iov_length(iov, count); + if (se->debug) { if (out->unique == 0) { fuse_log(FUSE_LOG_DEBUG, "NOTIFY: code=%d length=%u\n", @@ -278,12 +281,15 @@ static int fuse_send_msg(struct fuse_session *se, struct fuse_chan *ch, } } - err = _fuse_send_msg(se, ch, iov, count); + if (is_uring) + err = fuse_send_msg_uring(req, iov, count); + else + err = fuse_write_msg_dev(se, ch, iov, count); + trace_request_reply(out->unique, out->len, out->error, err); return err; } - int fuse_send_reply_iov_nofree(fuse_req_t req, int error, struct iovec *iov, int count) { @@ -305,7 +311,7 @@ int fuse_send_reply_iov_nofree(fuse_req_t req, int error, struct iovec *iov, iov[0].iov_base = &out; iov[0].iov_len = sizeof(struct fuse_out_header); - return fuse_send_msg(req->se, req->ch, iov, count); + return fuse_send_msg(req->se, req->ch, iov, count, req); } static int send_reply_iov(fuse_req_t req, int error, struct iovec *iov, @@ -321,6 +327,9 @@ static int send_reply_iov(fuse_req_t req, int error, struct iovec *iov, static int send_reply(fuse_req_t req, int error, const void *arg, size_t argsize) { + if (req->is_uring) + return send_reply_uring(req, error, arg, argsize); + struct iovec iov[2]; int count = 1; if (argsize) { @@ -599,7 +608,7 @@ static int fuse_send_data_iov_fallback(struct fuse_session *se, struct fuse_chan *ch, struct iovec *iov, int iov_count, struct fuse_bufvec *buf, - size_t len) + size_t len, fuse_req_t req) { struct fuse_bufvec mem_buf = FUSE_BUFVEC_INIT(len); void *mbuf; @@ -614,7 +623,7 @@ static int fuse_send_data_iov_fallback(struct fuse_session *se, iov[iov_count].iov_base = buf->buf[0].mem; iov[iov_count].iov_len = len; iov_count++; - return fuse_send_msg(se, ch, iov, iov_count); + return fuse_send_msg(se, ch, iov, iov_count, req); } res = posix_memalign(&mbuf, pagesize, len); @@ -632,7 +641,7 @@ static int fuse_send_data_iov_fallback(struct fuse_session *se, iov[iov_count].iov_base = mbuf; iov[iov_count].iov_len = len; iov_count++; - res = fuse_send_msg(se, ch, iov, iov_count); + res = fuse_send_msg(se, ch, iov, iov_count, req); free(mbuf); return res; @@ -764,8 +773,9 @@ static int grow_pipe_to_max(int pipefd) } static int fuse_send_data_iov(struct fuse_session *se, struct fuse_chan *ch, - struct iovec *iov, int iov_count, - struct fuse_bufvec *buf, unsigned int flags) + struct iovec *iov, int iov_count, + struct fuse_bufvec *buf, unsigned int flags, + fuse_req_t req) { int res; size_t len = fuse_buf_size(buf); @@ -917,7 +927,7 @@ static int fuse_send_data_iov(struct fuse_session *se, struct fuse_chan *ch, iov[iov_count].iov_base = mbuf; iov[iov_count].iov_len = len; iov_count++; - res = fuse_send_msg(se, ch, iov, iov_count); + res = fuse_send_msg(se, ch, iov, iov_count, req); free(mbuf); return res; } @@ -964,17 +974,17 @@ clear_pipe: return res; fallback: - return fuse_send_data_iov_fallback(se, ch, iov, iov_count, buf, len); + return fuse_send_data_iov_fallback(se, ch, iov, iov_count, buf, len, req); } #else static int fuse_send_data_iov(struct fuse_session *se, struct fuse_chan *ch, struct iovec *iov, int iov_count, - struct fuse_bufvec *buf, unsigned int flags) + struct fuse_bufvec *req_data, unsigned int flags) { - size_t len = fuse_buf_size(buf); + size_t len = fuse_buf_size(req_data); (void) flags; - return fuse_send_data_iov_fallback(se, ch, iov, iov_count, buf, len); + return fuse_send_data_iov_fallback(se, ch, iov, iov_count, req_data, len); } #endif @@ -985,13 +995,16 @@ int fuse_reply_data(fuse_req_t req, struct fuse_bufvec *bufv, struct fuse_out_header out; int res; + if (req->is_uring) + return fuse_reply_data_uring(req, bufv, flags); + iov[0].iov_base = &out; iov[0].iov_len = sizeof(struct fuse_out_header); out.unique = req->unique; out.error = 0; - res = fuse_send_data_iov(req->se, req->ch, iov, 1, bufv, flags); + res = fuse_send_data_iov(req->se, req->ch, iov, 1, bufv, flags, req); if (res <= 0) { fuse_free_req(req); return res; @@ -1743,8 +1756,8 @@ static void do_write_buf(fuse_req_t req, const fuse_ino_t nodeid, sizeof(struct fuse_write_in); } if (bufv.buf[0].size < arg->size) { - fuse_log(FUSE_LOG_ERR, "fuse: %s: buffer size too small\n", - __func__); + fuse_log(FUSE_LOG_ERR, + "fuse: %s: buffer size too small\n", __func__); fuse_reply_err(req, EIO); goto out; } @@ -2576,6 +2589,7 @@ _do_init(fuse_req_t req, const fuse_ino_t nodeid, const void *op_in, LL_SET_DEFAULT(se->op.readdirplus, FUSE_CAP_READDIRPLUS); LL_SET_DEFAULT(se->op.readdirplus && se->op.readdir, FUSE_CAP_READDIRPLUS_AUTO); + LL_SET_DEFAULT(1, FUSE_CAP_OVER_IO_URING); /* This could safely become default, but libfuse needs an API extension * to support it @@ -2840,6 +2854,7 @@ static int send_notify_iov(struct fuse_session *se, int notify_code, struct iovec *iov, int count) { struct fuse_out_header out; + struct fuse_req *req = NULL; if (!se->got_init) return -ENOTCONN; @@ -2849,7 +2864,7 @@ static int send_notify_iov(struct fuse_session *se, int notify_code, iov[0].iov_base = &out; iov[0].iov_len = sizeof(struct fuse_out_header); - return fuse_send_msg(se, NULL, iov, count); + return fuse_send_msg(se, NULL, iov, count, req); } int fuse_lowlevel_notify_poll(struct fuse_pollhandle *ph) @@ -2991,6 +3006,7 @@ int fuse_lowlevel_notify_store(struct fuse_session *se, fuse_ino_t ino, struct iovec iov[3]; size_t size = fuse_buf_size(bufv); int res; + struct fuse_req *req = NULL; if (!se) return -EINVAL; @@ -3011,7 +3027,7 @@ int fuse_lowlevel_notify_store(struct fuse_session *se, fuse_ino_t ino, iov[1].iov_base = &outarg; iov[1].iov_len = sizeof(outarg); - res = fuse_send_data_iov(se, NULL, iov, 2, bufv, flags); + res = fuse_send_data_iov(se, NULL, iov, 2, bufv, flags, req); if (res > 0) res = -res; @@ -3392,7 +3408,7 @@ void fuse_session_process_buf_internal(struct fuse_session *se, .iov_len = sizeof(struct fuse_out_header), }; - fuse_send_msg(se, ch, &iov, 1); + fuse_send_msg(se, ch, &iov, 1, NULL); goto clear_pipe; } diff --git a/lib/fuse_uring.c b/lib/fuse_uring.c index 65ee987..55d68fa 100644 --- a/lib/fuse_uring.c +++ b/lib/fuse_uring.c @@ -141,6 +141,156 @@ fuse_uring_sqe_prepare(struct io_uring_sqe *sqe, struct fuse_ring_ent *req, sqe->__pad1 = 0; } +static int fuse_uring_commit_sqe(struct fuse_ring_pool *ring_pool, + struct fuse_ring_queue *queue, + struct fuse_ring_ent *ring_ent) +{ + struct fuse_session *se = ring_pool->se; + struct fuse_uring_req_header *rrh = ring_ent->req_header; + struct fuse_out_header *out = (struct fuse_out_header *)&rrh->in_out; + struct fuse_uring_ent_in_out *ent_in_out = + (struct fuse_uring_ent_in_out *)&rrh->ring_ent_in_out; + struct io_uring_sqe *sqe = io_uring_get_sqe(&queue->ring); + + if (sqe == NULL) { + /* This is an impossible condition, unless there is a bug. + * The kernel sent back an SQEs, which is assigned to a request. + * There is no way to get out of SQEs, as the number of + * SQEs matches the number tof requests. + */ + + se->error = -EIO; + fuse_log(FUSE_LOG_ERR, "Failed to get a ring SQEs\n"); + + return -EIO; + } + + fuse_uring_sqe_prepare(sqe, ring_ent, + FUSE_IO_URING_CMD_COMMIT_AND_FETCH); + + fuse_uring_sqe_set_req_data(fuse_uring_get_sqe_cmd(sqe), queue->qid, + ring_ent->req_commit_id); + + if (se->debug) { + fuse_log(FUSE_LOG_DEBUG, " unique: %llu, result=%d\n", + out->unique, ent_in_out->payload_sz); + } + + /* XXX: This needs to be a ring config option */ + io_uring_submit(&queue->ring); + + return 0; +} + +int send_reply_uring(fuse_req_t req, int error, const void *arg, size_t argsize) +{ + int res; + struct fuse_ring_ent *ring_ent = + container_of(req, struct fuse_ring_ent, req); + struct fuse_uring_req_header *rrh = ring_ent->req_header; + struct fuse_out_header *out = (struct fuse_out_header *)&rrh->in_out; + struct fuse_uring_ent_in_out *ent_in_out = + (struct fuse_uring_ent_in_out *)&rrh->ring_ent_in_out; + + struct fuse_ring_queue *queue = ring_ent->ring_queue; + struct fuse_ring_pool *ring_pool = queue->ring_pool; + size_t max_payload_sz = ring_pool->max_req_payload_sz; + + if (argsize > max_payload_sz) { + fuse_log(FUSE_LOG_ERR, "argsize %zu exceeds buffer size %zu", + argsize, max_payload_sz); + error = -EINVAL; + } else if (argsize) { + memcpy(ring_ent->op_payload, arg, argsize); + } + ent_in_out->payload_sz = argsize; + + out->error = error; + out->unique = req->unique; + + res = fuse_uring_commit_sqe(ring_pool, queue, ring_ent); + + fuse_free_req(req); + + return res; +} + +int fuse_reply_data_uring(fuse_req_t req, struct fuse_bufvec *bufv, + enum fuse_buf_copy_flags flags) +{ + struct fuse_ring_ent *ring_ent = + container_of(req, struct fuse_ring_ent, req); + + struct fuse_ring_queue *queue = ring_ent->ring_queue; + struct fuse_ring_pool *ring_pool = queue->ring_pool; + struct fuse_uring_req_header *rrh = ring_ent->req_header; + struct fuse_out_header *out = (struct fuse_out_header *)&rrh->in_out; + struct fuse_uring_ent_in_out *ent_in_out = + (struct fuse_uring_ent_in_out *)&rrh->ring_ent_in_out; + size_t max_payload_sz = ring_ent->req_payload_sz; + struct fuse_bufvec dest_vec = FUSE_BUFVEC_INIT(max_payload_sz); + int res; + + dest_vec.buf[0].mem = ring_ent->op_payload; + dest_vec.buf[0].size = max_payload_sz; + + res = fuse_buf_copy(&dest_vec, bufv, flags); + + out->error = res < 0 ? res : 0; + out->unique = req->unique; + + ent_in_out->payload_sz = res > 0 ? res : 0; + + res = fuse_uring_commit_sqe(ring_pool, queue, ring_ent); + + fuse_free_req(req); + + return res; +} + +/** + * Copy the iov into the ring buffer and submit and commit/fetch sqe + */ +int fuse_send_msg_uring(fuse_req_t req, struct iovec *iov, int count) +{ + struct fuse_ring_ent *ring_ent = + container_of(req, struct fuse_ring_ent, req); + + struct fuse_ring_queue *queue = ring_ent->ring_queue; + struct fuse_ring_pool *ring_pool = queue->ring_pool; + struct fuse_uring_req_header *rrh = ring_ent->req_header; + struct fuse_out_header *out = (struct fuse_out_header *)&rrh->in_out; + struct fuse_uring_ent_in_out *ent_in_out = + (struct fuse_uring_ent_in_out *)&rrh->ring_ent_in_out; + size_t max_buf = ring_pool->max_req_payload_sz; + size_t len = 0; + int res = 0; + + /* copy iov into the payload, idx=0 is the header section */ + for (int idx = 1; idx < count; idx++) { + struct iovec *cur = &iov[idx]; + + if (len + cur->iov_len > max_buf) { + fuse_log(FUSE_LOG_ERR, + "iov[%d] exceeds buffer size %zu", + idx, max_buf); + res = -EINVAL; /* Gracefully handle this? */ + break; + } + + memcpy(ring_ent->op_payload + len, cur->iov_base, cur->iov_len); + len += cur->iov_len; + } + + ent_in_out->payload_sz = len; + + out->error = res; + out->unique = req->unique; + out->len = len; + + return fuse_uring_commit_sqe(ring_pool, queue, ring_ent); +} + static int fuse_queue_setup_io_uring(struct io_uring *ring, size_t qid, size_t depth, int fd, int evfd) { diff --git a/lib/fuse_uring_i.h b/lib/fuse_uring_i.h index e9f2989..40cda5b 100644 --- a/lib/fuse_uring_i.h +++ b/lib/fuse_uring_i.h @@ -27,6 +27,12 @@ struct fuse_in_header; int fuse_uring_start(struct fuse_session *se); int fuse_uring_stop(struct fuse_session *se); +int send_reply_uring(fuse_req_t req, int error, const void *arg, + size_t argsize); + +int fuse_reply_data_uring(fuse_req_t req, struct fuse_bufvec *bufv, + enum fuse_buf_copy_flags flags); +int fuse_send_msg_uring(fuse_req_t req, struct iovec *iov, int count); #else // HAVE_URING @@ -40,6 +46,29 @@ static inline int fuse_uring_stop(struct fuse_session *se FUSE_VAR_UNUSED) return -ENOTSUP; } +static inline int send_reply_uring(fuse_req_t req FUSE_VAR_UNUSED, + int error FUSE_VAR_UNUSED, + const void *arg FUSE_VAR_UNUSED, + size_t argsize FUSE_VAR_UNUSED) +{ + return -ENOTSUP; +} + +static inline int +fuse_reply_data_uring(fuse_req_t req FUSE_VAR_UNUSED, + struct fuse_bufvec *bufv FUSE_VAR_UNUSED, + enum fuse_buf_copy_flags flags FUSE_VAR_UNUSED) +{ + return -ENOTSUP; +} + +static inline int fuse_send_msg_uring(fuse_req_t req FUSE_VAR_UNUSED, + struct iovec *iov FUSE_VAR_UNUSED, + int count FUSE_VAR_UNUSED) +{ + return -ENOTSUP; +} + #endif // HAVE_URING #endif // FUSE_URING_I_H_ |