aboutsummaryrefslogtreecommitdiffstats
path: root/sources/xdma_driver/xdma/cdev_sgdma.c
diff options
context:
space:
mode:
Diffstat (limited to 'sources/xdma_driver/xdma/cdev_sgdma.c')
-rw-r--r--sources/xdma_driver/xdma/cdev_sgdma.c552
1 files changed, 552 insertions, 0 deletions
diff --git a/sources/xdma_driver/xdma/cdev_sgdma.c b/sources/xdma_driver/xdma/cdev_sgdma.c
new file mode 100644
index 0000000..410ef45
--- /dev/null
+++ b/sources/xdma_driver/xdma/cdev_sgdma.c
@@ -0,0 +1,552 @@
+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2016-present, Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include <asm/cacheflush.h>
+#include "libxdma_api.h"
+#include "xdma_cdev.h"
+#include "cdev_sgdma.h"
+
+/* Module Parameters */
+unsigned int sgdma_timeout = 10;
+module_param(sgdma_timeout, uint, 0644);
+MODULE_PARM_DESC(sgdma_timeout, "timeout in seconds for sgdma, default is 10 sec.");
+
+/*
+ * character device file operations for SG DMA engine
+ */
+static loff_t char_sgdma_llseek(struct file *file, loff_t off, int whence)
+{
+ loff_t newpos = 0;
+
+ switch (whence) {
+ case 0: /* SEEK_SET */
+ newpos = off;
+ break;
+ case 1: /* SEEK_CUR */
+ newpos = file->f_pos + off;
+ break;
+ case 2: /* SEEK_END, @TODO should work from end of address space */
+ newpos = UINT_MAX + off;
+ break;
+ default: /* can't happen */
+ return -EINVAL;
+ }
+ if (newpos < 0)
+ return -EINVAL;
+ file->f_pos = newpos;
+ dbg_fops("char_sgdma_llseek: pos=%lld\n", (signed long long)newpos);
+
+#if 0
+ pr_err("0x%p, off 0x%lld, whence %d -> pos %lld.\n",
+ file, (signed long long)off, whence, (signed long long)off);
+#endif
+
+ return newpos;
+}
+
+/* char_sgdma_read_write() -- Read from or write to the device
+ *
+ * @buf userspace buffer
+ * @count number of bytes in the userspace buffer
+ * @pos byte-address in device
+ * @dir_to_device If !0, a write to the device is performed
+ *
+ * Iterate over the userspace buffer, taking at most 255 * PAGE_SIZE bytes for
+ * each DMA transfer.
+ *
+ * For each transfer, get the user pages, build a sglist, map, build a
+ * descriptor table. submit the transfer. wait for the interrupt handler
+ * to wake us on completion.
+ */
+
+static int check_transfer_align(struct xdma_engine *engine,
+ const char __user *buf, size_t count, loff_t pos, int sync)
+{
+ BUG_ON(!engine);
+
+ /* AXI ST or AXI MM non-incremental addressing mode? */
+ if (engine->non_incr_addr) {
+ int buf_lsb = (int)((uintptr_t)buf) & (engine->addr_align - 1);
+ size_t len_lsb = count & ((size_t)engine->len_granularity - 1);
+ int pos_lsb = (int)pos & (engine->addr_align - 1);
+
+ dbg_tfr("AXI ST or MM non-incremental\n");
+ dbg_tfr("buf_lsb = %d, pos_lsb = %d, len_lsb = %ld\n", buf_lsb,
+ pos_lsb, len_lsb);
+
+ if (buf_lsb != 0) {
+ dbg_tfr("FAIL: non-aligned buffer address %p\n", buf);
+ return -EINVAL;
+ }
+
+ if ((pos_lsb != 0) && (sync)) {
+ dbg_tfr("FAIL: non-aligned AXI MM FPGA addr 0x%llx\n",
+ (unsigned long long)pos);
+ return -EINVAL;
+ }
+
+ if (len_lsb != 0) {
+ dbg_tfr("FAIL: len %d is not a multiple of %d\n",
+ (int)count,
+ (int)engine->len_granularity);
+ return -EINVAL;
+ }
+ /* AXI MM incremental addressing mode */
+ } else {
+ int buf_lsb = (int)((uintptr_t)buf) & (engine->addr_align - 1);
+ int pos_lsb = (int)pos & (engine->addr_align - 1);
+
+ if (buf_lsb != pos_lsb) {
+ dbg_tfr("FAIL: Misalignment error\n");
+ dbg_tfr("host addr %p, FPGA addr 0x%llx\n", buf, pos);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * Map a user memory range into a scatterlist
+ * inspired by vhost_scsi_map_to_sgl()
+ * Returns the number of scatterlist entries used or -errno on error.
+ */
+static inline void xdma_io_cb_release(struct xdma_io_cb *cb)
+{
+ int i;
+
+ for (i = 0; i < cb->pages_nr; i++)
+ put_page(cb->pages[i]);
+
+ sg_free_table(&cb->sgt);
+ kfree(cb->pages);
+
+ memset(cb, 0, sizeof(*cb));
+}
+
+static void char_sgdma_unmap_user_buf(struct xdma_io_cb *cb, bool write)
+{
+ int i;
+
+ sg_free_table(&cb->sgt);
+
+ if (!cb->pages || !cb->pages_nr)
+ return;
+
+ for (i = 0; i < cb->pages_nr; i++) {
+ if (cb->pages[i]) {
+ if (!write)
+ set_page_dirty_lock(cb->pages[i]);
+ put_page(cb->pages[i]);
+ } else
+ break;
+ }
+
+ if (i != cb->pages_nr)
+ pr_info("sgl pages %d/%u.\n", i, cb->pages_nr);
+
+ kfree(cb->pages);
+ cb->pages = NULL;
+}
+
+static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write)
+{
+ struct sg_table *sgt = &cb->sgt;
+ unsigned long len = cb->len;
+ char *buf = cb->buf;
+ struct scatterlist *sg;
+ unsigned int pages_nr = (((unsigned long)buf + len + PAGE_SIZE -1) -
+ ((unsigned long)buf & PAGE_MASK))
+ >> PAGE_SHIFT;
+ int i;
+ int rv;
+
+ if (pages_nr == 0) {
+ return -EINVAL;
+ }
+
+ if (sg_alloc_table(sgt, pages_nr, GFP_KERNEL)) {
+ pr_err("sgl OOM.\n");
+ return -ENOMEM;
+ }
+
+ cb->pages = kcalloc(pages_nr, sizeof(struct page *), GFP_KERNEL);
+ if (!cb->pages) {
+ pr_err("pages OOM.\n");
+ rv = -ENOMEM;
+ goto err_out;
+ }
+
+ rv = get_user_pages_fast((unsigned long)buf, pages_nr, 1/* write */,
+ cb->pages);
+ /* No pages were pinned */
+ if (rv < 0) {
+ pr_err("unable to pin down %u user pages, %d.\n",
+ pages_nr, rv);
+ goto err_out;
+ }
+ /* Less pages pinned than wanted */
+ if (rv != pages_nr) {
+ pr_err("unable to pin down all %u user pages, %d.\n",
+ pages_nr, rv);
+ rv = -EFAULT;
+ cb->pages_nr = rv;
+ goto err_out;
+ }
+
+ for (i = 1; i < pages_nr; i++) {
+ if (cb->pages[i - 1] == cb->pages[i]) {
+ pr_err("duplicate pages, %d, %d.\n",
+ i - 1, i);
+ rv = -EFAULT;
+ cb->pages_nr = pages_nr;
+ goto err_out;
+ }
+ }
+
+ sg = sgt->sgl;
+ for (i = 0; i < pages_nr; i++, sg = sg_next(sg)) {
+ //unsigned int offset = (uintptr_t)buf & ~PAGE_MASK;
+ unsigned int offset = offset_in_page(buf);
+ unsigned int nbytes = min_t(unsigned int, PAGE_SIZE - offset, len);
+
+ flush_dcache_page(cb->pages[i]);
+ sg_set_page(sg, cb->pages[i], nbytes, offset);
+
+ buf += nbytes;
+ len -= nbytes;
+ }
+
+ BUG_ON(len);
+ cb->pages_nr = pages_nr;
+ return 0;
+
+err_out:
+ char_sgdma_unmap_user_buf(cb, write);
+
+ return rv;
+}
+
+static ssize_t char_sgdma_read_write(struct file *file, char __user *buf,
+ size_t count, loff_t *pos, bool write)
+{
+ int rv;
+ ssize_t res = 0;
+ struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data;
+ struct xdma_dev *xdev;
+ struct xdma_engine *engine;
+ struct xdma_io_cb cb;
+
+ rv = xcdev_check(__func__, xcdev, 1);
+ if (rv < 0)
+ return rv;
+ xdev = xcdev->xdev;
+ engine = xcdev->engine;
+
+ dbg_tfr("file 0x%p, priv 0x%p, buf 0x%p,%llu, pos %llu, W %d, %s.\n",
+ file, file->private_data, buf, (u64)count, (u64)*pos, write,
+ engine->name);
+
+ if ((write && engine->dir != DMA_TO_DEVICE) ||
+ (!write && engine->dir != DMA_FROM_DEVICE)) {
+ pr_err("r/w mismatch. W %d, dir %d.\n",
+ write, engine->dir);
+ return -EINVAL;
+ }
+
+ rv = check_transfer_align(engine, buf, count, *pos, 1);
+ if (rv) {
+ pr_info("Invalid transfer alignment detected\n");
+ return rv;
+ }
+
+ memset(&cb, 0, sizeof(struct xdma_io_cb));
+ cb.buf = buf;
+ cb.len = count;
+ rv = char_sgdma_map_user_buf_to_sgl(&cb, write);
+ if (rv < 0)
+ return rv;
+
+ res = xdma_xfer_submit(xdev, engine->channel, write, *pos, &cb.sgt,
+ 0, sgdma_timeout * 1000);
+ //pr_err("xfer_submit return=%lld.\n", (s64)res);
+
+ //interrupt_status(xdev);
+
+ char_sgdma_unmap_user_buf(&cb, write);
+
+ return res;
+}
+
+
+static ssize_t char_sgdma_write(struct file *file, const char __user *buf,
+ size_t count, loff_t *pos)
+{
+ return char_sgdma_read_write(file, (char *)buf, count, pos, 1);
+}
+
+static ssize_t char_sgdma_read(struct file *file, char __user *buf,
+ size_t count, loff_t *pos)
+{
+ struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data;
+ struct xdma_engine *engine;
+ int rv;
+
+ rv = xcdev_check(__func__, xcdev, 1);
+ if (rv < 0)
+ return rv;
+
+ engine = xcdev->engine;
+
+ if (engine->streaming && engine->dir == DMA_FROM_DEVICE) {
+ rv = xdma_cyclic_transfer_setup(engine);
+ if (rv < 0 && rv != -EBUSY)
+ return rv;
+ /* 600 sec. timeout */
+ return xdma_engine_read_cyclic(engine, buf, count, 600000);
+ }
+
+ return char_sgdma_read_write(file, (char *)buf, count, pos, 0);
+}
+
+static int ioctl_do_perf_start(struct xdma_engine *engine, unsigned long arg)
+{
+ int rv;
+ struct xdma_dev *xdev;
+
+ BUG_ON(!engine);
+ xdev = engine->xdev;
+ BUG_ON(!xdev);
+
+ /* performance measurement already running on this engine? */
+ if (engine->xdma_perf) {
+ dbg_perf("IOCTL_XDMA_PERF_START failed!\n");
+ dbg_perf("Perf measurement already seems to be running!\n");
+ return -EBUSY;
+ }
+ engine->xdma_perf = kzalloc(sizeof(struct xdma_performance_ioctl),
+ GFP_KERNEL);
+
+ if (!engine->xdma_perf)
+ return -ENOMEM;
+
+ rv = copy_from_user(engine->xdma_perf,
+ (struct xdma_performance_ioctl *)arg,
+ sizeof(struct xdma_performance_ioctl));
+
+ if (rv < 0) {
+ dbg_perf("Failed to copy from user space 0x%lx\n", arg);
+ return -EINVAL;
+ }
+ if (engine->xdma_perf->version != IOCTL_XDMA_PERF_V1) {
+ dbg_perf("Unsupported IOCTL version %d\n",
+ engine->xdma_perf->version);
+ return -EINVAL;
+ }
+
+ enable_perf(engine);
+ dbg_perf("transfer_size = %d\n", engine->xdma_perf->transfer_size);
+ /* initialize wait queue */
+ init_waitqueue_head(&engine->xdma_perf_wq);
+ xdma_performance_submit(xdev, engine);
+
+ return 0;
+}
+
+static int ioctl_do_perf_stop(struct xdma_engine *engine, unsigned long arg)
+{
+ struct xdma_transfer *transfer = NULL;
+ int rv;
+
+ dbg_perf("IOCTL_XDMA_PERF_STOP\n");
+
+ /* no performance measurement running on this engine? */
+ if (!engine->xdma_perf) {
+ dbg_perf("No measurement in progress\n");
+ return -EINVAL;
+ }
+
+ /* stop measurement */
+ transfer = engine_cyclic_stop(engine);
+ dbg_perf("Waiting for measurement to stop\n");
+
+ if (engine->xdma_perf) {
+ get_perf_stats(engine);
+
+ rv = copy_to_user((void __user *)arg, engine->xdma_perf,
+ sizeof(struct xdma_performance_ioctl));
+ if (rv) {
+ dbg_perf("Error copying result to user\n");
+ return -EINVAL;
+ }
+ } else {
+ dbg_perf("engine->xdma_perf == NULL?\n");
+ }
+
+ kfree(engine->xdma_perf);
+ engine->xdma_perf = NULL;
+
+ return 0;
+}
+
+static int ioctl_do_perf_get(struct xdma_engine *engine, unsigned long arg)
+{
+ int rc;
+
+ BUG_ON(!engine);
+
+ dbg_perf("IOCTL_XDMA_PERF_GET\n");
+
+ if (engine->xdma_perf) {
+ get_perf_stats(engine);
+
+ rc = copy_to_user((void __user *)arg, engine->xdma_perf,
+ sizeof(struct xdma_performance_ioctl));
+ if (rc) {
+ dbg_perf("Error copying result to user\n");
+ return -EINVAL;
+ }
+ } else {
+ dbg_perf("engine->xdma_perf == NULL?\n");
+ return -EPROTO;
+ }
+
+ return 0;
+}
+
+static int ioctl_do_addrmode_set(struct xdma_engine *engine, unsigned long arg)
+{
+ return engine_addrmode_set(engine, arg);
+}
+
+static int ioctl_do_addrmode_get(struct xdma_engine *engine, unsigned long arg)
+{
+ int rv;
+ unsigned long src;
+
+ BUG_ON(!engine);
+ src = !!engine->non_incr_addr;
+
+ dbg_perf("IOCTL_XDMA_ADDRMODE_GET\n");
+ rv = put_user(src, (int __user *)arg);
+
+ return rv;
+}
+
+static int ioctl_do_align_get(struct xdma_engine *engine, unsigned long arg)
+{
+ BUG_ON(!engine);
+
+ dbg_perf("IOCTL_XDMA_ALIGN_GET\n");
+ return put_user(engine->addr_align, (int __user *)arg);
+}
+
+static long char_sgdma_ioctl(struct file *file, unsigned int cmd,
+ unsigned long arg)
+{
+ struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data;
+ struct xdma_dev *xdev;
+ struct xdma_engine *engine;
+
+ int rv = 0;
+
+ rv = xcdev_check(__func__, xcdev, 1);
+ if (rv < 0)
+ return rv;
+
+ xdev = xcdev->xdev;
+ engine = xcdev->engine;
+
+ switch (cmd) {
+ case IOCTL_XDMA_PERF_START:
+ rv = ioctl_do_perf_start(engine, arg);
+ break;
+ case IOCTL_XDMA_PERF_STOP:
+ rv = ioctl_do_perf_stop(engine, arg);
+ break;
+ case IOCTL_XDMA_PERF_GET:
+ rv = ioctl_do_perf_get(engine, arg);
+ break;
+ case IOCTL_XDMA_ADDRMODE_SET:
+ rv = ioctl_do_addrmode_set(engine, arg);
+ break;
+ case IOCTL_XDMA_ADDRMODE_GET:
+ rv = ioctl_do_addrmode_get(engine, arg);
+ break;
+ case IOCTL_XDMA_ALIGN_GET:
+ rv = ioctl_do_align_get(engine, arg);
+ break;
+ default:
+ dbg_perf("Unsupported operation\n");
+ rv = -EINVAL;
+ break;
+ }
+
+ return rv;
+}
+
+static int char_sgdma_open(struct inode *inode, struct file *file)
+{
+ struct xdma_cdev *xcdev;
+ struct xdma_engine *engine;
+
+ char_open(inode, file);
+
+ xcdev = (struct xdma_cdev *)file->private_data;
+ engine = xcdev->engine;
+
+ if (engine->streaming && engine->dir == DMA_FROM_DEVICE) {
+ if (engine->device_open == 1)
+ return -EBUSY;
+ else
+ engine->device_open = 1;
+ }
+
+ return 0;
+}
+
+static int char_sgdma_close(struct inode *inode, struct file *file)
+{
+ struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data;
+ struct xdma_engine *engine;
+ int rv;
+
+ rv = xcdev_check(__func__, xcdev, 1);
+ if (rv < 0)
+ return rv;
+
+ engine = xcdev->engine;
+
+ if (engine->streaming && engine->dir == DMA_FROM_DEVICE) {
+ engine->device_open = 0;
+ if (engine->cyclic_req)
+ return xdma_cyclic_transfer_teardown(engine);
+ }
+
+ return 0;
+}
+static const struct file_operations sgdma_fops = {
+ .owner = THIS_MODULE,
+ .open = char_sgdma_open,
+ .release = char_sgdma_close,
+ .write = char_sgdma_write,
+ .read = char_sgdma_read,
+ .unlocked_ioctl = char_sgdma_ioctl,
+ .llseek = char_sgdma_llseek,
+};
+
+void cdev_sgdma_init(struct xdma_cdev *xcdev)
+{
+ cdev_init(&xcdev->cdev, &sgdma_fops);
+}
generated by cgit v1.2.3 (git 2.39.1) at 2026-10-06 02:46:51 +0000