diff options
Diffstat (limited to 'sources/xdma_driver')
29 files changed, 13483 insertions, 0 deletions
diff --git a/sources/xdma_driver/.gitignore b/sources/xdma_driver/.gitignore new file mode 100644 index 0000000..951f319 --- /dev/null +++ b/sources/xdma_driver/.gitignore @@ -0,0 +1,9 @@ +xdma/*.cmd +xdma/*.d +xdma/*.o +xdma/*.ko +xdma/*.o.cmd +xdma/*.order +xdma/*.mod +xdma/*.mod.c +xdma/*.symvers
\ No newline at end of file diff --git a/sources/xdma_driver/COPYING b/sources/xdma_driver/COPYING new file mode 100644 index 0000000..3912109 --- /dev/null +++ b/sources/xdma_driver/COPYING @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/sources/xdma_driver/LICENSE b/sources/xdma_driver/LICENSE new file mode 100644 index 0000000..703e647 --- /dev/null +++ b/sources/xdma_driver/LICENSE @@ -0,0 +1,30 @@ +BSD License + +For Xilinx DMA IP software + +Copyright (c) 2016-present, Xilinx, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Xilinx nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/sources/xdma_driver/include/libxdma_api.h b/sources/xdma_driver/include/libxdma_api.h new file mode 100644 index 0000000..00d4355 --- /dev/null +++ b/sources/xdma_driver/include/libxdma_api.h @@ -0,0 +1,130 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef __XDMA_BASE_API_H__ +#define __XDMA_BASE_API_H__ + +#include <linux/types.h> +#include <linux/scatterlist.h> +#include <linux/interrupt.h> + +/* + * functions exported by the xdma driver + */ + +typedef struct { + u64 write_submitted; + u64 write_completed; + u64 read_requested; + u64 read_completed; + u64 restart; + u64 open; + u64 close; + u64 msix_trigger; +} xdma_statistics; + +/* + * This struct should be constantly updated by XMDA using u64_stats_* APIs + * The front end will read the structure without locking (That's why updating atomically is a must) + * every time it prints the statistics. + */ +//static XDMA_Statistics stats; + +/* + * xdma_device_open - read the pci bars and configure the fpga + * should be called from probe() + * NOTE: + * user interrupt will not enabled until xdma_user_isr_enable() + * is called + * @pdev: ptr to pci_dev + * @mod_name: the module name to be used for request_irq + * @user_max: max # of user/event (interrupts) to be configured + * @channel_max: max # of c2h and h2c channels to be configured + * NOTE: if the user/channel provisioned is less than the max specified, + * libxdma will update the user_max/channel_max + * returns + * a opaque handle (for libxdma to identify the device) + * NULL, in case of error + */ +void *xdma_device_open(const char *mod_name, struct pci_dev *pdev, + int *user_max, int *h2c_channel_max, int *c2h_channel_max); + +/* + * xdma_device_close - prepare fpga for removal: disable all interrupts (users + * and xdma) and release all resources + * should called from remove() + * @pdev: ptr to struct pci_dev + * @tuples: from xdma_device_open() + */ +void xdma_device_close(struct pci_dev *pdev, void *dev_handle); + +/* + * xdma_device_restart - restart the fpga + * @pdev: ptr to struct pci_dev + * TODO: + * may need more refining on the parameter list + * return < 0 in case of error + * TODO: exact error code will be defined later + */ +int xdma_device_restart(struct pci_dev *pdev, void *dev_handle); + +/* + * xdma_user_isr_register - register a user ISR handler + * It is expected that the xdma will register the ISR, and for the user + * interrupt, it will call the corresponding handle if it is registered and + * enabled. + * + * @pdev: ptr to the the pci_dev struct + * @mask: bitmask of user interrupts (0 ~ 15)to be registered + * bit 0: user interrupt 0 + * ... + * bit 15: user interrupt 15 + * any bit above bit 15 will be ignored. + * @handler: the correspoinding handler + * a NULL handler will be treated as de-registeration + * @name: to be passed to the handler, ignored if handler is NULL` + * @dev: to be passed to the handler, ignored if handler is NULL` + * return < 0 in case of error + * TODO: exact error code will be defined later + */ +int xdma_user_isr_register(void *dev_hndl, unsigned int mask, + irq_handler_t handler, void *dev); + +/* + * xdma_user_isr_enable/disable - enable or disable user interrupt + * @pdev: ptr to the the pci_dev struct + * @mask: bitmask of user interrupts (0 ~ 15)to be registered + * return < 0 in case of error + * TODO: exact error code will be defined later + */ +int xdma_user_isr_enable(void *dev_hndl, unsigned int mask); +int xdma_user_isr_disable(void *dev_hndl, unsigned int mask); + +/* + * xdma_xfer_submit - submit data for dma operation (for both read and write) + * This is a blocking call + * @channel: channle number (< channel_max) + * == channel_max means libxdma can pick any channel available:q + + * @dir: DMA_FROM/TO_DEVICE + * @offset: offset into the DDR/BRAM memory to read from or write to + * @sg_tbl: the scatter-gather list of data buffers + * @timeout: timeout in mili-seconds, *currently ignored + * return # of bytes transfered or + * < 0 in case of error + * TODO: exact error code will be defined later + */ +ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, + struct sg_table *sgt, bool dma_mapped, int timeout_ms); + + +#endif diff --git a/sources/xdma_driver/libxdma/Makefile b/sources/xdma_driver/libxdma/Makefile new file mode 100644 index 0000000..1003f0f --- /dev/null +++ b/sources/xdma_driver/libxdma/Makefile @@ -0,0 +1,25 @@ +SHELL = /bin/bash + +topdir := $(shell cd $(src)/.. && pwd) + +TARGET_MODULE:=libxdma + +EXTRA_CFLAGS := -I$(topdir)/include +EXTRA_CFLAGS += -D__LIBXDMA_MOD__ + +ifneq ($(KERNELRELEASE),) + obj-m := $(TARGET_MODULE).o +# $(TARGET_MODULE)-objs := libxdma.o +else + BUILDSYSTEM_DIR:=/lib/modules/$(shell uname -r)/build + PWD:=$(shell pwd) +all : + $(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) modules + +clean: + $(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) clean + +install: all + $(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) modules_install + +endif diff --git a/sources/xdma_driver/libxdma/libxdma.c b/sources/xdma_driver/libxdma/libxdma.c new file mode 100644 index 0000000..b9884d7 --- /dev/null +++ b/sources/xdma_driver/libxdma/libxdma.c @@ -0,0 +1,4442 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/string.h> +#include <linux/mm.h> +#include <linux/errno.h> +#include <linux/sched.h> +#include <linux/vmalloc.h> + +#include "libxdma.h" +#include "libxdma_api.h" +#include "cdev_sgdma.h" + +/* SECTION: Module licensing */ + +#ifdef __LIBXDMA_MOD__ +#include "version.h" +#define DRV_MODULE_NAME "libxdma" +#define DRV_MODULE_DESC "Xilinx XDMA Base Driver" +#define DRV_MODULE_RELDATE "Feb. 2018" + +static char version[] = + DRV_MODULE_DESC " " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n"; + +MODULE_AUTHOR("Xilinx, Inc."); +MODULE_DESCRIPTION(DRV_MODULE_DESC); +MODULE_VERSION(DRV_MODULE_VERSION); +MODULE_LICENSE("Dual BSD/GPL"); +#endif + +/* Module Parameters */ +static unsigned int poll_mode; +module_param(poll_mode, uint, 0644); +MODULE_PARM_DESC(poll_mode, "Set 1 for hw polling, default is 0 (interrupts)"); + +static unsigned int interrupt_mode; +module_param(interrupt_mode, uint, 0644); +MODULE_PARM_DESC(interrupt_mode, "0 - MSI-x , 1 - MSI, 2 - Legacy"); + +static unsigned int enable_credit_mp; +module_param(enable_credit_mp, uint, 0644); +MODULE_PARM_DESC(enable_credit_mp, "Set 1 to enable creidt feature, default is 0 (no credit control)"); + +unsigned int desc_blen_max = XDMA_DESC_BLEN_MAX; +module_param(desc_blen_max, uint, 0644); +MODULE_PARM_DESC(desc_blen_max, "per descriptor max. buffer length, default is (1 << 28) - 1"); + +/* + * xdma device management + * maintains a list of the xdma devices + */ +static LIST_HEAD(xdev_list); +static DEFINE_MUTEX(xdev_mutex); + +static LIST_HEAD(xdev_rcu_list); +static DEFINE_SPINLOCK(xdev_rcu_lock); + +#ifndef list_last_entry +#define list_last_entry(ptr, type, member) \ + list_entry((ptr)->prev, type, member) +#endif + +static inline void xdev_list_add(struct xdma_dev *xdev) +{ + mutex_lock(&xdev_mutex); + if (list_empty(&xdev_list)) + xdev->idx = 0; + else { + struct xdma_dev *last; + + last = list_last_entry(&xdev_list, struct xdma_dev, list_head); + xdev->idx = last->idx + 1; + } + list_add_tail(&xdev->list_head, &xdev_list); + mutex_unlock(&xdev_mutex); + + dbg_init("dev %s, xdev 0x%p, xdma idx %d.\n", + dev_name(&xdev->pdev->dev), xdev, xdev->idx); + + spin_lock(&xdev_rcu_lock); + list_add_tail_rcu(&xdev->rcu_node, &xdev_rcu_list); + spin_unlock(&xdev_rcu_lock); +} + +#undef list_last_entry + +static inline void xdev_list_remove(struct xdma_dev *xdev) +{ + mutex_lock(&xdev_mutex); + list_del(&xdev->list_head); + mutex_unlock(&xdev_mutex); + + spin_lock(&xdev_rcu_lock); + list_del_rcu(&xdev->rcu_node); + spin_unlock(&xdev_rcu_lock); + synchronize_rcu(); +} + +struct xdma_dev *xdev_find_by_pdev(struct pci_dev *pdev) +{ + struct xdma_dev *xdev, *tmp; + + mutex_lock(&xdev_mutex); + list_for_each_entry_safe(xdev, tmp, &xdev_list, list_head) { + if (xdev->pdev == pdev) { + mutex_unlock(&xdev_mutex); + return xdev; + } + } + mutex_unlock(&xdev_mutex); + return NULL; +} +EXPORT_SYMBOL_GPL(xdev_find_by_pdev); + +static inline int debug_check_dev_hndl(const char *fname, struct pci_dev *pdev, + void *hndl) +{ + struct xdma_dev *xdev; + + if (!pdev) + return -EINVAL; + + xdev = xdev_find_by_pdev(pdev); + if (!xdev) { + pr_info("%s pdev 0x%p, hndl 0x%p, NO match found!\n", + fname, pdev, hndl); + return -EINVAL; + } + if (xdev != hndl) { + pr_err("%s pdev 0x%p, hndl 0x%p != 0x%p!\n", + fname, pdev, hndl, xdev); + return -EINVAL; + } + + return 0; +} + +#ifdef __LIBXDMA_DEBUG__ +/* SECTION: Function definitions */ +inline void __write_register(const char *fn, u32 value, void *iomem, unsigned long off) +{ + pr_err("%s: w reg 0x%lx(0x%p), 0x%x.\n", fn, off, iomem, value); + iowrite32(value, iomem); +} +#define write_register(v,mem,off) __write_register(__func__, v, mem, off) +#else +#define write_register(v,mem,off) iowrite32(v, mem) +#endif + +inline u32 read_register(void *iomem) +{ + return ioread32(iomem); +} + +static inline u32 build_u32(u32 hi, u32 lo) +{ + return ((hi & 0xFFFFUL) << 16) | (lo & 0xFFFFUL); +} + +static inline u64 build_u64(u64 hi, u64 lo) +{ + return ((hi & 0xFFFFFFFULL) << 32) | (lo & 0xFFFFFFFFULL); +} + +static void check_nonzero_interrupt_status(struct xdma_dev *xdev) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + u32 w; + + w = read_register(®->user_int_enable); + if (w) + pr_info("%s xdma%d user_int_enable = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + + w = read_register(®->channel_int_enable); + if (w) + pr_info("%s xdma%d channel_int_enable = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + + w = read_register(®->user_int_request); + if (w) + pr_info("%s xdma%d user_int_request = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + w = read_register(®->channel_int_request); + if (w) + pr_info("%s xdma%d channel_int_request = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + + w = read_register(®->user_int_pending); + if (w) + pr_info("%s xdma%d user_int_pending = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + w = read_register(®->channel_int_pending); + if (w) + pr_info("%s xdma%d channel_int_pending = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); +} + +/* channel_interrupts_enable -- Enable interrupts we are interested in */ +static void channel_interrupts_enable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->channel_int_enable_w1s, XDMA_OFS_INT_CTRL); +} + +/* channel_interrupts_disable -- Disable interrupts we not interested in */ +static void channel_interrupts_disable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->channel_int_enable_w1c, XDMA_OFS_INT_CTRL); +} + +/* user_interrupts_enable -- Enable interrupts we are interested in */ +static void user_interrupts_enable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->user_int_enable_w1s, XDMA_OFS_INT_CTRL); +} + +/* user_interrupts_disable -- Disable interrupts we not interested in */ +static void user_interrupts_disable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->user_int_enable_w1c, XDMA_OFS_INT_CTRL); +} + +/* read_interrupts -- Print the interrupt controller status */ +static u32 read_interrupts(struct xdma_dev *xdev) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + u32 lo; + u32 hi; + + /* extra debugging; inspect complete engine set of registers */ + hi = read_register(®->user_int_request); + dbg_io("ioread32(0x%p) returned 0x%08x (user_int_request).\n", + ®->user_int_request, hi); + lo = read_register(®->channel_int_request); + dbg_io("ioread32(0x%p) returned 0x%08x (channel_int_request)\n", + ®->channel_int_request, lo); + + /* return interrupts: user in upper 16-bits, channel in lower 16-bits */ + return build_u32(hi, lo); +} + +void enable_perf(struct xdma_engine *engine) +{ + u32 w; + + w = XDMA_PERF_CLEAR; + write_register(w, &engine->regs->perf_ctrl, + (unsigned long)(&engine->regs->perf_ctrl) - + (unsigned long)(&engine->regs)); + read_register(&engine->regs->identifier); + w = XDMA_PERF_AUTO | XDMA_PERF_RUN; + write_register(w, &engine->regs->perf_ctrl, + (unsigned long)(&engine->regs->perf_ctrl) - + (unsigned long)(&engine->regs)); + read_register(&engine->regs->identifier); + + dbg_perf("IOCTL_XDMA_PERF_START\n"); + +} +EXPORT_SYMBOL_GPL(enable_perf); + +void get_perf_stats(struct xdma_engine *engine) +{ + u32 hi; + u32 lo; + + BUG_ON(!engine); + + if (!engine->xdma_perf) { + pr_info("%s perf struct not set up.\n", engine->name); + return; + } + + hi = 0; + lo = read_register(&engine->regs->completed_desc_count); + engine->xdma_perf->iterations = build_u64(hi, lo); + + hi = read_register(&engine->regs->perf_cyc_hi); + lo = read_register(&engine->regs->perf_cyc_lo); + + engine->xdma_perf->clock_cycle_count = build_u64(hi, lo); + + hi = read_register(&engine->regs->perf_dat_hi); + lo = read_register(&engine->regs->perf_dat_lo); + engine->xdma_perf->data_cycle_count = build_u64(hi, lo); + + hi = read_register(&engine->regs->perf_pnd_hi); + lo = read_register(&engine->regs->perf_pnd_lo); + engine->xdma_perf->pending_count = build_u64(hi, lo); +} +EXPORT_SYMBOL_GPL(get_perf_stats); + +static void engine_reg_dump(struct xdma_engine *engine) +{ + u32 w; + + BUG_ON(!engine); + + w = read_register(&engine->regs->identifier); + pr_info("%s: ioread32(0x%p) = 0x%08x (id).\n", + engine->name, &engine->regs->identifier, w); + w &= BLOCK_ID_MASK; + if (w != BLOCK_ID_HEAD) { + pr_info("%s: engine id missing, 0x%08x exp. & 0x%x = 0x%x\n", + engine->name, w, BLOCK_ID_MASK, BLOCK_ID_HEAD); + return; + } + /* extra debugging; inspect complete engine set of registers */ + w = read_register(&engine->regs->status); + pr_info("%s: ioread32(0x%p) = 0x%08x (status).\n", + engine->name, &engine->regs->status, w); + w = read_register(&engine->regs->control); + pr_info("%s: ioread32(0x%p) = 0x%08x (control)\n", + engine->name, &engine->regs->control, w); + w = read_register(&engine->sgdma_regs->first_desc_lo); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_lo)\n", + engine->name, &engine->sgdma_regs->first_desc_lo, w); + w = read_register(&engine->sgdma_regs->first_desc_hi); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_hi)\n", + engine->name, &engine->sgdma_regs->first_desc_hi, w); + w = read_register(&engine->sgdma_regs->first_desc_adjacent); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_adjacent).\n", + engine->name, &engine->sgdma_regs->first_desc_adjacent, w); + w = read_register(&engine->regs->completed_desc_count); + pr_info("%s: ioread32(0x%p) = 0x%08x (completed_desc_count).\n", + engine->name, &engine->regs->completed_desc_count, w); + w = read_register(&engine->regs->interrupt_enable_mask); + pr_info("%s: ioread32(0x%p) = 0x%08x (interrupt_enable_mask)\n", + engine->name, &engine->regs->interrupt_enable_mask, w); +} + +/** + * engine_status_read() - read status of SG DMA engine (optionally reset) + * + * Stores status in engine->status. + * + * @return -1 on failure, status register otherwise + */ +static void engine_status_dump(struct xdma_engine *engine) +{ + u32 v = engine->status; + char buffer[256]; + char *buf = buffer; + int len = 0; + + len = sprintf(buf, "SG engine %s status: 0x%08x: ", engine->name, v); + + if ((v & XDMA_STAT_BUSY)) + len += sprintf(buf + len, "BUSY,"); + if ((v & XDMA_STAT_DESC_STOPPED)) + len += sprintf(buf + len, "DESC_STOPPED,"); + if ((v & XDMA_STAT_DESC_COMPLETED)) + len += sprintf(buf + len, "DESC_COMPL,"); + + /* common H2C & C2H */ + if ((v & XDMA_STAT_COMMON_ERR_MASK)) { + if ((v & XDMA_STAT_ALIGN_MISMATCH)) + len += sprintf(buf + len, "ALIGN_MISMATCH "); + if ((v & XDMA_STAT_MAGIC_STOPPED)) + len += sprintf(buf + len, "MAGIC_STOPPED "); + if ((v & XDMA_STAT_INVALID_LEN)) + len += sprintf(buf + len, "INVLIAD_LEN "); + if ((v & XDMA_STAT_IDLE_STOPPED)) + len += sprintf(buf + len, "IDLE_STOPPED "); + buf[len - 1] = ','; + } + + if ((engine->dir == DMA_TO_DEVICE)) { + /* H2C only */ + if ((v & XDMA_STAT_H2C_R_ERR_MASK)) { + len += sprintf(buf + len, "R:"); + if ((v & XDMA_STAT_H2C_R_UNSUPP_REQ)) + len += sprintf(buf + len, "UNSUPP_REQ "); + if ((v & XDMA_STAT_H2C_R_COMPL_ABORT)) + len += sprintf(buf + len, "COMPL_ABORT "); + if ((v & XDMA_STAT_H2C_R_PARITY_ERR)) + len += sprintf(buf + len, "PARITY "); + if ((v & XDMA_STAT_H2C_R_HEADER_EP)) + len += sprintf(buf + len, "HEADER_EP "); + if ((v & XDMA_STAT_H2C_R_UNEXP_COMPL)) + len += sprintf(buf + len, "UNEXP_COMPL "); + buf[len - 1] = ','; + } + + if ((v & XDMA_STAT_H2C_W_ERR_MASK)) { + len += sprintf(buf + len, "W:"); + if ((v & XDMA_STAT_H2C_W_DECODE_ERR)) + len += sprintf(buf + len, "DECODE_ERR "); + if ((v & XDMA_STAT_H2C_W_SLAVE_ERR)) + len += sprintf(buf + len, "SLAVE_ERR "); + buf[len - 1] = ','; + } + + } else { + /* C2H only */ + if ((v & XDMA_STAT_C2H_R_ERR_MASK)) { + len += sprintf(buf + len, "R:"); + if ((v & XDMA_STAT_C2H_R_DECODE_ERR)) + len += sprintf(buf + len, "DECODE_ERR "); + if ((v & XDMA_STAT_C2H_R_SLAVE_ERR)) + len += sprintf(buf + len, "SLAVE_ERR "); + buf[len - 1] = ','; + } + } + + /* common H2C & C2H */ + if ((v & XDMA_STAT_DESC_ERR_MASK)) { + len += sprintf(buf + len, "DESC_ERR:"); + if ((v & XDMA_STAT_DESC_UNSUPP_REQ)) + len += sprintf(buf + len, "UNSUPP_REQ "); + if ((v & XDMA_STAT_DESC_COMPL_ABORT)) + len += sprintf(buf + len, "COMPL_ABORT "); + if ((v & XDMA_STAT_DESC_PARITY_ERR)) + len += sprintf(buf + len, "PARITY "); + if ((v & XDMA_STAT_DESC_HEADER_EP)) + len += sprintf(buf + len, "HEADER_EP "); + if ((v & XDMA_STAT_DESC_UNEXP_COMPL)) + len += sprintf(buf + len, "UNEXP_COMPL "); + buf[len - 1] = ','; + } + + buf[len - 1] = '\0'; + pr_info("%s\n", buffer); +} + +static u32 engine_status_read(struct xdma_engine *engine, bool clear, bool dump) +{ + u32 value; + + BUG_ON(!engine); + + if (dump) + engine_reg_dump(engine); + + /* read status register */ + if (clear) + value = engine->status = + read_register(&engine->regs->status_rc); + else + value = engine->status = read_register(&engine->regs->status); + + if (dump) + engine_status_dump(engine); + + return value; +} + +/** + * xdma_engine_stop() - stop an SG DMA engine + * + */ +static void xdma_engine_stop(struct xdma_engine *engine) +{ + u32 w; + + BUG_ON(!engine); + dbg_tfr("xdma_engine_stop(engine=%p)\n", engine); + + w = 0; + w |= (u32)XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + w |= (u32)XDMA_CTRL_IE_MAGIC_STOPPED; + w |= (u32)XDMA_CTRL_IE_READ_ERROR; + w |= (u32)XDMA_CTRL_IE_DESC_ERROR; + + if (poll_mode) { + w |= (u32) XDMA_CTRL_POLL_MODE_WB; + } else { + w |= (u32)XDMA_CTRL_IE_DESC_STOPPED; + w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED; + + /* Disable IDLE STOPPED for MM */ + if ((engine->streaming && (engine->dir == DMA_FROM_DEVICE)) || + (engine->xdma_perf)) + w |= (u32)XDMA_CTRL_IE_IDLE_STOPPED; + } + + dbg_tfr("Stopping SG DMA %s engine; writing 0x%08x to 0x%p.\n", + engine->name, w, (u32 *)&engine->regs->control); + write_register(w, &engine->regs->control, + (unsigned long)(&engine->regs->control) - + (unsigned long)(&engine->regs)); + /* dummy read of status register to flush all previous writes */ + dbg_tfr("xdma_engine_stop(%s) done\n", engine->name); +} + +static void engine_start_mode_config(struct xdma_engine *engine) +{ + u32 w; + + BUG_ON(!engine); + + /* If a perf test is running, enable the engine interrupts */ + if (engine->xdma_perf) { + w = XDMA_CTRL_IE_DESC_STOPPED; + w |= XDMA_CTRL_IE_DESC_COMPLETED; + w |= XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + w |= XDMA_CTRL_IE_MAGIC_STOPPED; + w |= XDMA_CTRL_IE_IDLE_STOPPED; + w |= XDMA_CTRL_IE_READ_ERROR; + w |= XDMA_CTRL_IE_DESC_ERROR; + + write_register(w, &engine->regs->interrupt_enable_mask, + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); + } + + /* write control register of SG DMA engine */ + w = (u32)XDMA_CTRL_RUN_STOP; + w |= (u32)XDMA_CTRL_IE_READ_ERROR; + w |= (u32)XDMA_CTRL_IE_DESC_ERROR; + w |= (u32)XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + w |= (u32)XDMA_CTRL_IE_MAGIC_STOPPED; + + if (poll_mode) { + w |= (u32)XDMA_CTRL_POLL_MODE_WB; + } else { + w |= (u32)XDMA_CTRL_IE_DESC_STOPPED; + w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED; + + if ((engine->streaming && (engine->dir == DMA_FROM_DEVICE)) || + (engine->xdma_perf)) + w |= (u32)XDMA_CTRL_IE_IDLE_STOPPED; + + /* set non-incremental addressing mode */ + if (engine->non_incr_addr) + w |= (u32)XDMA_CTRL_NON_INCR_ADDR; + } + + dbg_tfr("iowrite32(0x%08x to 0x%p) (control)\n", w, + (void *)&engine->regs->control); + /* start the engine */ + write_register(w, &engine->regs->control, + (unsigned long)(&engine->regs->control) - + (unsigned long)(&engine->regs)); + + /* dummy read of status register to flush all previous writes */ + w = read_register(&engine->regs->status); + dbg_tfr("ioread32(0x%p) = 0x%08x (dummy read flushes writes).\n", + &engine->regs->status, w); +} + +/** + * engine_start() - start an idle engine with its first transfer on queue + * + * The engine will run and process all transfers that are queued using + * transfer_queue() and thus have their descriptor lists chained. + * + * During the run, new transfers will be processed if transfer_queue() has + * chained the descriptors before the hardware fetches the last descriptor. + * A transfer that was chained too late will invoke a new run of the engine + * initiated from the engine_service() routine. + * + * The engine must be idle and at least one transfer must be queued. + * This function does not take locks; the engine spinlock must already be + * taken. + * + */ +static struct xdma_transfer *engine_start(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer; + u32 w; + int extra_adj = 0; + + /* engine must be idle */ + BUG_ON(engine->running); + /* engine transfer queue must not be empty */ + BUG_ON(list_empty(&engine->transfer_list)); + /* inspect first transfer queued on the engine */ + transfer = list_entry(engine->transfer_list.next, struct xdma_transfer, + entry); + BUG_ON(!transfer); + + /* engine is no longer shutdown */ + engine->shutdown = ENGINE_SHUTDOWN_NONE; + + dbg_tfr("engine_start(%s): transfer=0x%p.\n", engine->name, transfer); + + /* initialize number of descriptors of dequeued transfers */ + engine->desc_dequeued = 0; + + /* write lower 32-bit of bus address of transfer first descriptor */ + w = cpu_to_le32(PCI_DMA_L(transfer->desc_bus)); + dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_lo)\n", w, + (void *)&engine->sgdma_regs->first_desc_lo); + write_register(w, &engine->sgdma_regs->first_desc_lo, + (unsigned long)(&engine->sgdma_regs->first_desc_lo) - + (unsigned long)(&engine->sgdma_regs)); + /* write upper 32-bit of bus address of transfer first descriptor */ + w = cpu_to_le32(PCI_DMA_H(transfer->desc_bus)); + dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_hi)\n", w, + (void *)&engine->sgdma_regs->first_desc_hi); + write_register(w, &engine->sgdma_regs->first_desc_hi, + (unsigned long)(&engine->sgdma_regs->first_desc_hi) - + (unsigned long)(&engine->sgdma_regs)); + + if (transfer->desc_adjacent > 0) { + extra_adj = transfer->desc_adjacent - 1; + if (extra_adj > MAX_EXTRA_ADJ) + extra_adj = MAX_EXTRA_ADJ; + } + dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_adjacent)\n", + extra_adj, (void *)&engine->sgdma_regs->first_desc_adjacent); + write_register(extra_adj, &engine->sgdma_regs->first_desc_adjacent, + (unsigned long)(&engine->sgdma_regs->first_desc_adjacent) - + (unsigned long)(&engine->sgdma_regs)); + + dbg_tfr("ioread32(0x%p) (dummy read flushes writes).\n", + &engine->regs->status); + mmiowb(); + + engine_start_mode_config(engine); + + engine_status_read(engine, 0, 0); + + dbg_tfr("%s engine 0x%p now running\n", engine->name, engine); + /* remember the engine is running */ + engine->running = 1; + return transfer; +} + +/** + * engine_service() - service an SG DMA engine + * + * must be called with engine->lock already acquired + * + * @engine pointer to struct xdma_engine + * + */ +static void engine_service_shutdown(struct xdma_engine *engine) +{ + /* if the engine stopped with RUN still asserted, de-assert RUN now */ + dbg_tfr("engine just went idle, resetting RUN_STOP.\n"); + xdma_engine_stop(engine); + engine->running = 0; + + /* awake task on engine's shutdown wait queue */ + wake_up_interruptible(&engine->shutdown_wq); +} + +struct xdma_transfer *engine_transfer_completion(struct xdma_engine *engine, + struct xdma_transfer *transfer) +{ + BUG_ON(!engine); + + if (unlikely(!transfer)) { + pr_info("%s: xfer empty.\n", engine->name); + return NULL; + } + + /* synchronous I/O? */ + /* awake task on transfer's wait queue */ + wake_up_interruptible(&transfer->wq); + + return transfer; +} + +struct xdma_transfer *engine_service_transfer_list(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 *pdesc_completed) +{ + BUG_ON(!engine); + BUG_ON(!pdesc_completed); + + if (unlikely(!transfer)) { + pr_info("%s xfer empty, pdesc completed %u.\n", + engine->name, *pdesc_completed); + return NULL; + } + + /* + * iterate over all the transfers completed by the engine, + * except for the last (i.e. use > instead of >=). + */ + while (transfer && (!transfer->cyclic) && + (*pdesc_completed > transfer->desc_num)) { + /* remove this transfer from pdesc_completed */ + *pdesc_completed -= transfer->desc_num; + dbg_tfr("%s engine completed non-cyclic xfer 0x%p (%d desc)\n", + engine->name, transfer, transfer->desc_num); + /* remove completed transfer from list */ + list_del(engine->transfer_list.next); + /* add to dequeued number of descriptors during this run */ + engine->desc_dequeued += transfer->desc_num; + /* mark transfer as succesfully completed */ + transfer->state = TRANSFER_STATE_COMPLETED; + + /* Complete transfer - sets transfer to NULL if an async + * transfer has completed */ + transfer = engine_transfer_completion(engine, transfer); + + /* if exists, get the next transfer on the list */ + if (!list_empty(&engine->transfer_list)) { + transfer = list_entry(engine->transfer_list.next, + struct xdma_transfer, entry); + dbg_tfr("Non-completed transfer %p\n", transfer); + } else { + /* no further transfers? */ + transfer = NULL; + } + } + + return transfer; +} + +static void engine_err_handle(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 desc_completed) +{ + u32 value; + + /* + * The BUSY bit is expected to be clear now but older HW has a race + * condition which could cause it to be still set. If it's set, re-read + * and check again. If it's still set, log the issue. + */ + if (engine->status & XDMA_STAT_BUSY) { + value = read_register(&engine->regs->status); + if ((value & XDMA_STAT_BUSY) && printk_ratelimit()) + pr_info("%s has errors but is still BUSY\n", + engine->name); + } + + if (printk_ratelimit()) { + pr_info("%s, s 0x%x, aborted xfer 0x%p, cmpl %d/%d\n", + engine->name, engine->status, transfer, desc_completed, + transfer->desc_num); + } + + /* mark transfer as failed */ + transfer->state = TRANSFER_STATE_FAILED; + xdma_engine_stop(engine); +} + +struct xdma_transfer *engine_service_final_transfer(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 *pdesc_completed) +{ + BUG_ON(!engine); + BUG_ON(!pdesc_completed); + + /* inspect the current transfer */ + if (unlikely(!transfer)) { + pr_info("%s xfer empty, pdesc completed %u.\n", + engine->name, *pdesc_completed); + return NULL; + } else { + if (((engine->dir == DMA_FROM_DEVICE) && + (engine->status & XDMA_STAT_C2H_ERR_MASK)) || + ((engine->dir == DMA_TO_DEVICE) && + (engine->status & XDMA_STAT_H2C_ERR_MASK))) { + pr_info("engine %s, status error 0x%x.\n", + engine->name, engine->status); + engine_status_dump(engine); + engine_err_handle(engine, transfer, *pdesc_completed); + goto transfer_del; + } + + if (engine->status & XDMA_STAT_BUSY) + pr_debug("engine %s is unexpectedly busy - ignoring\n", + engine->name); + + /* the engine stopped on current transfer? */ + if (*pdesc_completed < transfer->desc_num) { + transfer->state = TRANSFER_STATE_FAILED; + pr_info("%s, xfer 0x%p, stopped half-way, %d/%d.\n", + engine->name, transfer, *pdesc_completed, + transfer->desc_num); + } else { + dbg_tfr("engine %s completed transfer\n", engine->name); + dbg_tfr("Completed transfer ID = 0x%p\n", transfer); + dbg_tfr("*pdesc_completed=%d, transfer->desc_num=%d", + *pdesc_completed, transfer->desc_num); + + if (!transfer->cyclic) { + /* + * if the engine stopped on this transfer, + * it should be the last + */ + WARN_ON(*pdesc_completed > transfer->desc_num); + } + /* mark transfer as succesfully completed */ + transfer->state = TRANSFER_STATE_COMPLETED; + } + +transfer_del: + /* remove completed transfer from list */ + list_del(engine->transfer_list.next); + /* add to dequeued number of descriptors during this run */ + engine->desc_dequeued += transfer->desc_num; + + /* + * Complete transfer - sets transfer to NULL if an asynchronous + * transfer has completed + */ + transfer = engine_transfer_completion(engine, transfer); + } + + return transfer; +} + +static void engine_service_perf(struct xdma_engine *engine, u32 desc_completed) +{ + BUG_ON(!engine); + + /* performance measurement is running? */ + if (engine->xdma_perf) { + /* a descriptor was completed? */ + if (engine->status & XDMA_STAT_DESC_COMPLETED) { + engine->xdma_perf->iterations = desc_completed; + dbg_perf("transfer->xdma_perf->iterations=%d\n", + engine->xdma_perf->iterations); + } + + /* a descriptor stopped the engine? */ + if (engine->status & XDMA_STAT_DESC_STOPPED) { + engine->xdma_perf->stopped = 1; + /* + * wake any XDMA_PERF_IOCTL_STOP waiting for + * the performance run to finish + */ + wake_up_interruptible(&engine->xdma_perf_wq); + dbg_perf("transfer->xdma_perf stopped\n"); + } + } +} + +static void engine_transfer_dequeue(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer; + + BUG_ON(!engine); + + /* pick first transfer on the queue (was submitted to the engine) */ + transfer = list_entry(engine->transfer_list.next, struct xdma_transfer, + entry); + if (!transfer || transfer != &engine->cyclic_req->xfer) { + pr_info("%s, xfer 0x%p != 0x%p.\n", + engine->name, transfer, &engine->cyclic_req->xfer); + return; + } + dbg_tfr("%s engine completed cyclic transfer 0x%p (%d desc).\n", + engine->name, transfer, transfer->desc_num); + /* remove completed transfer from list */ + list_del(engine->transfer_list.next); +} + +static int engine_ring_process(struct xdma_engine *engine) +{ + struct xdma_result *result; + int start; + int eop_count = 0; + + BUG_ON(!engine); + result = engine->cyclic_result; + BUG_ON(!result); + + /* where we start receiving in the ring buffer */ + start = engine->rx_tail; + + /* iterate through all newly received RX result descriptors */ + dbg_tfr("%s, result %d, 0x%x, len 0x%x.\n", + engine->name, engine->rx_tail, result[engine->rx_tail].status, + result[engine->rx_tail].length); + while (result[engine->rx_tail].status && !engine->rx_overrun) { + /* EOP bit set in result? */ + if (result[engine->rx_tail].status & RX_STATUS_EOP){ + eop_count++; + } + + /* increment tail pointer */ + engine->rx_tail = (engine->rx_tail + 1) % CYCLIC_RX_PAGES_MAX; + + dbg_tfr("%s, head %d, tail %d, 0x%x, len 0x%x.\n", + engine->name, engine->rx_head, engine->rx_tail, + result[engine->rx_tail].status, + result[engine->rx_tail].length); + + /* overrun? */ + if (engine->rx_tail == engine->rx_head) { + dbg_tfr("%s: overrun\n", engine->name); + /* flag to user space that overrun has occurred */ + engine->rx_overrun = 1; + } + } + + return eop_count; +} + +static int engine_service_cyclic_polled(struct xdma_engine *engine) +{ + int eop_count = 0; + int rc = 0; + struct xdma_poll_wb *writeback_data; + u32 sched_limit = 0; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + writeback_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + + while (eop_count == 0) { + if (sched_limit != 0) { + if ((sched_limit % NUM_POLLS_PER_SCHED) == 0) + schedule(); + } + sched_limit++; + + /* Monitor descriptor writeback address for errors */ + if ((writeback_data->completed_desc_count) & WB_ERR_MASK) { + rc = -1; + break; + } + + eop_count = engine_ring_process(engine); + } + + if (eop_count == 0) { + engine_status_read(engine, 1, 0); + if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) + engine_transfer_dequeue(engine); + + engine_service_shutdown(engine); + } + } + + return rc; +} + +static int engine_service_cyclic_interrupt(struct xdma_engine *engine) +{ + int eop_count = 0; + struct xdma_transfer *xfer; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + engine_status_read(engine, 1, 0); + + eop_count = engine_ring_process(engine); + /* + * wake any reader on EOP, as one or more packets are now in + * the RX buffer + */ + xfer = &engine->cyclic_req->xfer; + if(enable_credit_mp){ + if (eop_count > 0) { + //engine->eop_found = 1; + } + wake_up_interruptible(&xfer->wq); + }else{ + if (eop_count > 0) { + /* awake task on transfer's wait queue */ + dbg_tfr("wake_up_interruptible() due to %d EOP's\n", eop_count); + engine->eop_found = 1; + wake_up_interruptible(&xfer->wq); + } + } + + /* engine was running but is no longer busy? */ + if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) + engine_transfer_dequeue(engine); + + engine_service_shutdown(engine); + } + + return 0; +} + +/* must be called with engine->lock already acquired */ +static int engine_service_cyclic(struct xdma_engine *engine) +{ + int rc = 0; + + dbg_tfr("engine_service_cyclic()"); + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + if (poll_mode) + rc = engine_service_cyclic_polled(engine); + else + rc = engine_service_cyclic_interrupt(engine); + + return rc; +} + + +static void engine_service_resume(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer_started; + + BUG_ON(!engine); + + /* engine stopped? */ + if (!engine->running) { + /* in the case of shutdown, let it finish what's in the Q */ + if (!list_empty(&engine->transfer_list)) { + /* (re)start engine */ + transfer_started = engine_start(engine); + pr_info("re-started %s engine with pending xfer 0x%p\n", + engine->name, transfer_started); + /* engine was requested to be shutdown? */ + } else if (engine->shutdown & ENGINE_SHUTDOWN_REQUEST) { + engine->shutdown |= ENGINE_SHUTDOWN_IDLE; + /* awake task on engine's shutdown wait queue */ + wake_up_interruptible(&engine->shutdown_wq); + } else { + dbg_tfr("no pending transfers, %s engine stays idle.\n", + engine->name); + } + } else { + /* engine is still running? */ + if (list_empty(&engine->transfer_list)) { + pr_warn("no queued transfers but %s engine running!\n", + engine->name); + WARN_ON(1); + } + } +} + +/** + * engine_service() - service an SG DMA engine + * + * must be called with engine->lock already acquired + * + * @engine pointer to struct xdma_engine + * + */ +static int engine_service(struct xdma_engine *engine, int desc_writeback) +{ + struct xdma_transfer *transfer = NULL; + u32 desc_count = desc_writeback & WB_COUNT_MASK; + u32 err_flag = desc_writeback & WB_ERR_MASK; + int rv = 0; + struct xdma_poll_wb *wb_data; + + BUG_ON(!engine); + + /* If polling detected an error, signal to the caller */ + if (err_flag) + rv = -1; + + /* Service the engine */ + if (!engine->running) { + dbg_tfr("Engine was not running!!! Clearing status\n"); + engine_status_read(engine, 1, 0); + return 0; + } + + /* + * If called by the ISR or polling detected an error, read and clear + * engine status. For polled mode descriptor completion, this read is + * unnecessary and is skipped to reduce latency + */ + if ((desc_count == 0) || (err_flag != 0)) + engine_status_read(engine, 1, 0); + + /* + * engine was running but is no longer busy, or writeback occurred, + * shut down + */ + if ((engine->running && !(engine->status & XDMA_STAT_BUSY)) || + (desc_count != 0)) + engine_service_shutdown(engine); + + /* + * If called from the ISR, or if an error occurred, the descriptor + * count will be zero. In this scenario, read the descriptor count + * from HW. In polled mode descriptor completion, this read is + * unnecessary and is skipped to reduce latency + */ + if (!desc_count) + desc_count = read_register(&engine->regs->completed_desc_count); + dbg_tfr("desc_count = %d\n", desc_count); + + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) { + /* pick first transfer on queue (was submitted to the engine) */ + transfer = list_entry(engine->transfer_list.next, + struct xdma_transfer, entry); + + dbg_tfr("head of queue transfer 0x%p has %d descriptors\n", + transfer, (int)transfer->desc_num); + + dbg_tfr("Engine completed %d desc, %d not yet dequeued\n", + (int)desc_count, + (int)desc_count - engine->desc_dequeued); + + engine_service_perf(engine, desc_count); + } + + /* account for already dequeued transfers during this engine run */ + desc_count -= engine->desc_dequeued; + + /* Process all but the last transfer */ + transfer = engine_service_transfer_list(engine, transfer, &desc_count); + + /* + * Process final transfer - includes checks of number of descriptors to + * detect faulty completion + */ + transfer = engine_service_final_transfer(engine, transfer, &desc_count); + + /* Before starting engine again, clear the writeback data */ + if (poll_mode) { + wb_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + wb_data->completed_desc_count = 0; + } + + /* Restart the engine following the servicing */ + engine_service_resume(engine); + + return 0; +} + +/* engine_service_work */ +static void engine_service_work(struct work_struct *work) +{ + struct xdma_engine *engine; + unsigned long flags; + + engine = container_of(work, struct xdma_engine, work); + BUG_ON(engine->magic != MAGIC_ENGINE); + + /* lock the engine */ + spin_lock_irqsave(&engine->lock, flags); + + dbg_tfr("engine_service() for %s engine %p\n", + engine->name, engine); + if (engine->cyclic_req) + engine_service_cyclic(engine); + else + engine_service(engine, 0); + + /* re-enable interrupts for this engine */ + if (engine->xdev->msix_enabled){ + write_register(engine->interrupt_enable_mask_value, + &engine->regs->interrupt_enable_mask_w1s, + (unsigned long)(&engine->regs->interrupt_enable_mask_w1s) - + (unsigned long)(&engine->regs)); + } else + channel_interrupts_enable(engine->xdev, engine->irq_bitmask); + + /* unlock the engine */ + spin_unlock_irqrestore(&engine->lock, flags); +} + +static u32 engine_service_wb_monitor(struct xdma_engine *engine, + u32 expected_wb) +{ + struct xdma_poll_wb *wb_data; + u32 desc_wb = 0; + u32 sched_limit = 0; + unsigned long timeout; + + BUG_ON(!engine); + wb_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + + /* + * Poll the writeback location for the expected number of + * descriptors / error events This loop is skipped for cyclic mode, + * where the expected_desc_count passed in is zero, since it cannot be + * determined before the function is called + */ + + timeout = jiffies + (POLL_TIMEOUT_SECONDS * HZ); + while (expected_wb != 0) { + desc_wb = wb_data->completed_desc_count; + + if (desc_wb & WB_ERR_MASK) + break; + else if (desc_wb == expected_wb) + break; + + /* RTO - prevent system from hanging in polled mode */ + if (time_after(jiffies, timeout)) { + dbg_tfr("Polling timeout occurred"); + dbg_tfr("desc_wb = 0x%08x, expected 0x%08x\n", desc_wb, + expected_wb); + if ((desc_wb & WB_COUNT_MASK) > expected_wb) + desc_wb = expected_wb | WB_ERR_MASK; + + break; + } + + /* + * Define NUM_POLLS_PER_SCHED to limit how much time is spent + * in the scheduler + */ + + if (sched_limit != 0) { + if ((sched_limit % NUM_POLLS_PER_SCHED) == 0) + schedule(); + } + sched_limit++; + } + + return desc_wb; +} + +static int engine_service_poll(struct xdma_engine *engine, + u32 expected_desc_count) +{ + struct xdma_poll_wb *writeback_data; + u32 desc_wb = 0; + unsigned long flags; + int rv = 0; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + writeback_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + + if ((expected_desc_count & WB_COUNT_MASK) != expected_desc_count) { + dbg_tfr("Queued descriptor count is larger than supported\n"); + return -1; + } + + /* + * Poll the writeback location for the expected number of + * descriptors / error events This loop is skipped for cyclic mode, + * where the expected_desc_count passed in is zero, since it cannot be + * determined before the function is called + */ + + desc_wb = engine_service_wb_monitor(engine, expected_desc_count); + + spin_lock_irqsave(&engine->lock, flags); + dbg_tfr("%s service.\n", engine->name); + if (engine->cyclic_req) { + rv = engine_service_cyclic(engine); + } else { + rv = engine_service(engine, desc_wb); + } + spin_unlock_irqrestore(&engine->lock, flags); + + return rv; +} + +static irqreturn_t user_irq_service(int irq, struct xdma_user_irq *user_irq) +{ + unsigned long flags; + + BUG_ON(!user_irq); + + if (user_irq->handler) + return user_irq->handler(user_irq->user_idx, user_irq->dev); + + spin_lock_irqsave(&(user_irq->events_lock), flags); + if (!user_irq->events_irq) { + user_irq->events_irq = 1; + wake_up_interruptible(&(user_irq->events_wq)); + } + spin_unlock_irqrestore(&(user_irq->events_lock), flags); + + return IRQ_HANDLED; +} + +/* + * xdma_isr() - Interrupt handler + * + * @dev_id pointer to xdma_dev + */ +static irqreturn_t xdma_isr(int irq, void *dev_id) +{ + u32 ch_irq; + u32 user_irq; + u32 mask; + struct xdma_dev *xdev; + struct interrupt_regs *irq_regs; + + dbg_irq("(irq=%d, dev 0x%p) <<<< ISR.\n", irq, dev_id); + BUG_ON(!dev_id); + xdev = (struct xdma_dev *)dev_id; + + if (!xdev) { + WARN_ON(!xdev); + dbg_irq("xdma_isr(irq=%d) xdev=%p ??\n", irq, xdev); + return IRQ_NONE; + } + + irq_regs = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + + /* read channel interrupt requests */ + ch_irq = read_register(&irq_regs->channel_int_request); + dbg_irq("ch_irq = 0x%08x\n", ch_irq); + + /* + * disable all interrupts that fired; these are re-enabled individually + * after the causing module has been fully serviced. + */ + if (ch_irq) + channel_interrupts_disable(xdev, ch_irq); + + /* read user interrupts - this read also flushes the above write */ + user_irq = read_register(&irq_regs->user_int_request); + dbg_irq("user_irq = 0x%08x\n", user_irq); + + if (user_irq) { + int user = 0; + u32 mask = 1; + int max = xdev->h2c_channel_max; + + for (; user < max && user_irq; user++, mask <<= 1) { + if (user_irq & mask) { + user_irq &= ~mask; + user_irq_service(irq, &xdev->user_irq[user]); + } + } + } + + mask = ch_irq & xdev->mask_irq_h2c; + if (mask) { + int channel = 0; + int max = xdev->h2c_channel_max; + + /* iterate over H2C (PCIe read) */ + for (channel = 0; channel < max && mask; channel++) { + struct xdma_engine *engine = &xdev->engine_h2c[channel]; + + /* engine present and its interrupt fired? */ + if((engine->irq_bitmask & mask) && + (engine->magic == MAGIC_ENGINE)) { + mask &= ~engine->irq_bitmask; + dbg_tfr("schedule_work, %s.\n", engine->name); + schedule_work(&engine->work); + } + } + } + + mask = ch_irq & xdev->mask_irq_c2h; + if (mask) { + int channel = 0; + int max = xdev->c2h_channel_max; + + /* iterate over C2H (PCIe write) */ + for (channel = 0; channel < max && mask; channel++) { + struct xdma_engine *engine = &xdev->engine_c2h[channel]; + + /* engine present and its interrupt fired? */ + if((engine->irq_bitmask & mask) && + (engine->magic == MAGIC_ENGINE)) { + mask &= ~engine->irq_bitmask; + dbg_tfr("schedule_work, %s.\n", engine->name); + schedule_work(&engine->work); + } + } + } + + xdev->irq_count++; + return IRQ_HANDLED; +} + +/* + * xdma_user_irq() - Interrupt handler for user interrupts in MSI-X mode + * + * @dev_id pointer to xdma_dev + */ +static irqreturn_t xdma_user_irq(int irq, void *dev_id) +{ + struct xdma_user_irq *user_irq; + + dbg_irq("(irq=%d) <<<< INTERRUPT SERVICE ROUTINE\n", irq); + + BUG_ON(!dev_id); + user_irq = (struct xdma_user_irq *)dev_id; + + return user_irq_service(irq, user_irq); +} + +/* + * xdma_channel_irq() - Interrupt handler for channel interrupts in MSI-X mode + * + * @dev_id pointer to xdma_dev + */ +static irqreturn_t xdma_channel_irq(int irq, void *dev_id) +{ + struct xdma_dev *xdev; + struct xdma_engine *engine; + struct interrupt_regs *irq_regs; + + dbg_irq("(irq=%d) <<<< INTERRUPT service ROUTINE\n", irq); + BUG_ON(!dev_id); + + engine = (struct xdma_engine *)dev_id; + xdev = engine->xdev; + + if (!xdev) { + WARN_ON(!xdev); + dbg_irq("xdma_channel_irq(irq=%d) xdev=%p ??\n", irq, xdev); + return IRQ_NONE; + } + + irq_regs = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + + /* Disable the interrupt for this engine */ + write_register(engine->interrupt_enable_mask_value, + &engine->regs->interrupt_enable_mask_w1c, + (unsigned long) + (&engine->regs->interrupt_enable_mask_w1c) - + (unsigned long)(&engine->regs)); + /* Dummy read to flush the above write */ + read_register(&irq_regs->channel_int_pending); + /* Schedule the bottom half */ + schedule_work(&engine->work); + + /* + * RTO - need to protect access here if multiple MSI-X are used for + * user interrupts + */ + xdev->irq_count++; + return IRQ_HANDLED; +} + +/* + * Unmap the BAR regions that had been mapped earlier using map_bars() + */ +static void unmap_bars(struct xdma_dev *xdev, struct pci_dev *dev) +{ + int i; + + for (i = 0; i < XDMA_BAR_NUM; i++) { + /* is this BAR mapped? */ + if (xdev->bar[i]) { + /* unmap BAR */ + pci_iounmap(dev, xdev->bar[i]); + /* mark as unmapped */ + xdev->bar[i] = NULL; + } + } +} + +static int map_single_bar(struct xdma_dev *xdev, struct pci_dev *dev, int idx) +{ + resource_size_t bar_start; + resource_size_t bar_len; + resource_size_t map_len; + + bar_start = pci_resource_start(dev, idx); + bar_len = pci_resource_len(dev, idx); + map_len = bar_len; + + xdev->bar[idx] = NULL; + + /* do not map BARs with length 0. Note that start MAY be 0! */ + if (!bar_len) { + //pr_info("BAR #%d is not present - skipping\n", idx); + return 0; + } + + /* BAR size exceeds maximum desired mapping? */ + if (bar_len > INT_MAX) { + pr_info("Limit BAR %d mapping from %llu to %d bytes\n", idx, + (u64)bar_len, INT_MAX); + map_len = (resource_size_t)INT_MAX; + } + /* + * map the full device memory or IO region into kernel virtual + * address space + */ + dbg_init("BAR%d: %llu bytes to be mapped.\n", idx, (u64)map_len); + xdev->bar[idx] = pci_iomap(dev, idx, map_len); + + if (!xdev->bar[idx]) { + pr_info("Could not map BAR %d.\n", idx); + return -1; + } + + pr_info("BAR%d at 0x%llx mapped at 0x%p, length=%llu(/%llu)\n", idx, + (u64)bar_start, xdev->bar[idx], (u64)map_len, (u64)bar_len); + + return (int)map_len; +} + +static int is_config_bar(struct xdma_dev *xdev, int idx) +{ + u32 irq_id = 0; + u32 cfg_id = 0; + int flag = 0; + u32 mask = 0xffff0000; /* Compare only XDMA ID's not Version number */ + struct interrupt_regs *irq_regs = + (struct interrupt_regs *) (xdev->bar[idx] + XDMA_OFS_INT_CTRL); + struct config_regs *cfg_regs = + (struct config_regs *)(xdev->bar[idx] + XDMA_OFS_CONFIG); + + irq_id = read_register(&irq_regs->identifier); + cfg_id = read_register(&cfg_regs->identifier); + + if (((irq_id & mask)== IRQ_BLOCK_ID) && + ((cfg_id & mask)== CONFIG_BLOCK_ID)) { + dbg_init("BAR %d is the XDMA config BAR\n", idx); + flag = 1; + } else { + dbg_init("BAR %d is NOT the XDMA config BAR: 0x%x, 0x%x.\n", + idx, irq_id, cfg_id); + flag = 0; + } + + return flag; +} + +static void identify_bars(struct xdma_dev *xdev, int *bar_id_list, int num_bars, + int config_bar_pos) +{ + /* + * The following logic identifies which BARs contain what functionality + * based on the position of the XDMA config BAR and the number of BARs + * detected. The rules are that the user logic and bypass logic BARs + * are optional. When both are present, the XDMA config BAR will be the + * 2nd BAR detected (config_bar_pos = 1), with the user logic being + * detected first and the bypass being detected last. When one is + * omitted, the type of BAR present can be identified by whether the + * XDMA config BAR is detected first or last. When both are omitted, + * only the XDMA config BAR is present. This somewhat convoluted + * approach is used instead of relying on BAR numbers in order to work + * correctly with both 32-bit and 64-bit BARs. + */ + + BUG_ON(!xdev); + BUG_ON(!bar_id_list); + + dbg_init("xdev 0x%p, bars %d, config at %d.\n", + xdev, num_bars, config_bar_pos); + + switch (num_bars) { + case 1: + /* Only one BAR present - no extra work necessary */ + break; + + case 2: + if (config_bar_pos == 0) { + xdev->bypass_bar_idx = bar_id_list[1]; + } else if (config_bar_pos == 1) { + xdev->user_bar_idx = bar_id_list[0]; + } else { + pr_info("2, XDMA config BAR unexpected %d.\n", + config_bar_pos); + } + break; + + case 3: + case 4: + if ((config_bar_pos == 1) || (config_bar_pos == 2)) { + /* user bar at bar #0 */ + xdev->user_bar_idx = bar_id_list[0]; + /* bypass bar at the last bar */ + xdev->bypass_bar_idx = bar_id_list[num_bars - 1]; + } else { + pr_info("3/4, XDMA config BAR unexpected %d.\n", + config_bar_pos); + } + break; + + default: + /* Should not occur - warn user but safe to continue */ + pr_info("Unexpected # BARs (%d), XDMA config BAR only.\n", + num_bars); + break; + + } + pr_info("%d BARs: config %d, user %d, bypass %d.\n", + num_bars, config_bar_pos, xdev->user_bar_idx, + xdev->bypass_bar_idx); +} + +/* map_bars() -- map device regions into kernel virtual address space + * + * Map the device memory regions into kernel virtual address space after + * verifying their sizes respect the minimum sizes needed + */ +static int map_bars(struct xdma_dev *xdev, struct pci_dev *dev) +{ + int rv; + int i; + int bar_id_list[XDMA_BAR_NUM]; + int bar_id_idx = 0; + int config_bar_pos = 0; + + /* iterate through all the BARs */ + for (i = 0; i < XDMA_BAR_NUM; i++) { + int bar_len; + + bar_len = map_single_bar(xdev, dev, i); + if (bar_len == 0) { + continue; + } else if (bar_len < 0) { + rv = -EINVAL; + goto fail; + } + + /* Try to identify BAR as XDMA control BAR */ + if ((bar_len >= XDMA_BAR_SIZE) && (xdev->config_bar_idx < 0)) { + + if (is_config_bar(xdev, i)) { + xdev->config_bar_idx = i; + config_bar_pos = bar_id_idx; + pr_info("config bar %d, pos %d.\n", + xdev->config_bar_idx, config_bar_pos); + } + } + + bar_id_list[bar_id_idx] = i; + bar_id_idx++; + } + + /* The XDMA config BAR must always be present */ + if (xdev->config_bar_idx < 0) { + pr_info("Failed to detect XDMA config BAR\n"); + rv = -EINVAL; + goto fail; + } + +#ifdef __LIBXDMA_CONFIG_BAR_ONLY__ + /* unmapped all other bars, except XDMA config. bar */ + for (i = 0; i < XDMA_BAR_NUM; i++) { + if (i == xdev->config_bar_idx) + continue; + + /* is this BAR mapped? */ + if (xdev->bar[i]) { + /* unmap BAR */ + pci_iounmap(dev, xdev->bar[i]); + /* mark as unmapped */ + xdev->bar[i] = NULL; + pr_info("unmap non-config bar %d.\n", i); + } + } +#else + identify_bars(xdev, bar_id_list, bar_id_idx, config_bar_pos); +#endif + + /* successfully mapped all required BAR regions */ + return 0; + +fail: + /* unwind; unmap any BARs that we did map */ + unmap_bars(xdev, dev); + return rv; +} + +/* + * MSI-X interrupt: + * <h2c+c2h channel_max> vectors, followed by <user_max> vectors + */ + +/* + * RTO - code to detect if MSI/MSI-X capability exists is derived + * from linux/pci/msi.c - pci_msi_check_device + */ + +#ifndef arch_msi_check_device +int arch_msi_check_device(struct pci_dev *dev, int nvec, int type) +{ + return 0; +} +#endif + +/* type = PCI_CAP_ID_MSI or PCI_CAP_ID_MSIX */ +static int msi_msix_capable(struct pci_dev *dev, int type) +{ + struct pci_bus *bus; + int ret; + + if (!dev || dev->no_msi) + return 0; + + for (bus = dev->bus; bus; bus = bus->parent) + if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) + return 0; + + ret = arch_msi_check_device(dev, 1, type); + if (ret) + return 0; + + if (!pci_find_capability(dev, type)) + return 0; + + return 1; +} + +static void disable_msi_msix(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + if (xdev->msix_enabled) { + pci_disable_msix(pdev); + xdev->msix_enabled = 0; + } else if (xdev->msi_enabled) { + pci_disable_msi(pdev); + xdev->msi_enabled = 0; + } +} + +static int enable_msi_msix(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + int rv = 0; + + BUG_ON(!xdev); + BUG_ON(!pdev); + + if (!interrupt_mode && msi_msix_capable(pdev, PCI_CAP_ID_MSIX)) { + int req_nvec = xdev->c2h_channel_max + xdev->h2c_channel_max + + xdev->user_max; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + dbg_init("Enabling MSI-X\n"); + rv = pci_alloc_irq_vectors(pdev, req_nvec, req_nvec, + PCI_IRQ_MSIX); +#else + int i; + + dbg_init("Enabling MSI-X\n"); + for (i = 0; i < req_nvec; i++) + xdev->entry[i].entry = i; + + rv = pci_enable_msix(pdev, xdev->entry, req_nvec); +#endif + if (rv < 0) + dbg_init("Couldn't enable MSI-X mode: %d\n", rv); + + xdev->msix_enabled = 1; + + } else if (interrupt_mode == 1 && + msi_msix_capable(pdev, PCI_CAP_ID_MSI)) { + /* enable message signalled interrupts */ + dbg_init("pci_enable_msi()\n"); + rv = pci_enable_msi(pdev); + if (rv < 0) + dbg_init("Couldn't enable MSI mode: %d\n", rv); + xdev->msi_enabled = 1; + + } else { + dbg_init("MSI/MSI-X not detected - using legacy interrupts\n"); + } + + return rv; +} + +static void pci_check_intr_pend(struct pci_dev *pdev) +{ + u16 v; + + pci_read_config_word(pdev, PCI_STATUS, &v); + if (v & PCI_STATUS_INTERRUPT) { + pr_info("%s PCI STATUS Interrupt pending 0x%x.\n", + dev_name(&pdev->dev), v); + pci_write_config_word(pdev, PCI_STATUS, PCI_STATUS_INTERRUPT); + } +} + +static void pci_keep_intx_enabled(struct pci_dev *pdev) +{ + /* workaround to a h/w bug: + * when msix/msi become unavaile, default to legacy. + * However the legacy enable was not checked. + * If the legacy was disabled, no ack then everything stuck + */ + u16 pcmd, pcmd_new; + + pci_read_config_word(pdev, PCI_COMMAND, &pcmd); + pcmd_new = pcmd & ~PCI_COMMAND_INTX_DISABLE; + if (pcmd_new != pcmd) { + pr_info("%s: clear INTX_DISABLE, 0x%x -> 0x%x.\n", + dev_name(&pdev->dev), pcmd, pcmd_new); + pci_write_config_word(pdev, PCI_COMMAND, pcmd_new); + } +} + +static void prog_irq_msix_user(struct xdma_dev *xdev, bool clear) +{ + /* user */ + struct interrupt_regs *int_regs = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + u32 i = xdev->c2h_channel_max + xdev->h2c_channel_max; + u32 max = i + xdev->user_max; + int j; + + for (j = 0; i < max; j++) { + u32 val = 0; + int k; + int shift = 0; + + if (clear) + i += 4; + else + for (k = 0; k < 4 && i < max; i++, k++, shift += 8) + val |= (i & 0x1f) << shift; + + write_register(val, &int_regs->user_msi_vector[j], + XDMA_OFS_INT_CTRL + + ((unsigned long)&int_regs->user_msi_vector[j] - + (unsigned long)int_regs)); + + dbg_init("vector %d, 0x%x.\n", j, val); + } +} + +static void prog_irq_msix_channel(struct xdma_dev *xdev, bool clear) +{ + struct interrupt_regs *int_regs = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + u32 max = xdev->c2h_channel_max + xdev->h2c_channel_max; + u32 i; + int j; + + /* engine */ + for (i = 0, j = 0; i < max; j++) { + u32 val = 0; + int k; + int shift = 0; + + if (clear) + i += 4; + else + for (k = 0; k < 4 && i < max; i++, k++, shift += 8) + val |= (i & 0x1f) << shift; + + write_register(val, &int_regs->channel_msi_vector[j], + XDMA_OFS_INT_CTRL + + ((unsigned long)&int_regs->channel_msi_vector[j] - + (unsigned long)int_regs)); + dbg_init("vector %d, 0x%x.\n", j, val); + } +} + +static void irq_msix_channel_teardown(struct xdma_dev *xdev) +{ + struct xdma_engine *engine; + int j = 0; + int i = 0; + + if (!xdev->msix_enabled) + return; + + prog_irq_msix_channel(xdev, 1); + + engine = xdev->engine_h2c; + for (i = 0; i < xdev->h2c_channel_max; i++, j++, engine++) { + if (!engine->msix_irq_line) + break; + dbg_sg("Release IRQ#%d for engine %p\n", engine->msix_irq_line, + engine); + free_irq(engine->msix_irq_line, engine); + } + + engine = xdev->engine_c2h; + for (i = 0; i < xdev->c2h_channel_max; i++, j++, engine++) { + if (!engine->msix_irq_line) + break; + dbg_sg("Release IRQ#%d for engine %p\n", engine->msix_irq_line, + engine); + free_irq(engine->msix_irq_line, engine); + } +} + +static int irq_msix_channel_setup(struct xdma_dev *xdev) +{ + int i; + int j = xdev->h2c_channel_max; + int rv = 0; + u32 vector; + struct xdma_engine *engine; + + BUG_ON(!xdev); + if (!xdev->msix_enabled) + return 0; + + engine = xdev->engine_h2c; + for (i = 0; i < xdev->h2c_channel_max; i++, engine++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + vector = pci_irq_vector(xdev->pdev, i); +#else + vector = xdev->entry[i].vector; +#endif + rv = request_irq(vector, xdma_channel_irq, 0, xdev->mod_name, + engine); + if (rv) { + pr_info("requesti irq#%d failed %d, engine %s.\n", + vector, rv, engine->name); + return rv; + } + pr_info("engine %s, irq#%d.\n", engine->name, vector); + engine->msix_irq_line = vector; + } + + engine = xdev->engine_c2h; + for (i = 0; i < xdev->c2h_channel_max; i++, j++, engine++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + vector = pci_irq_vector(xdev->pdev, j); +#else + vector = xdev->entry[j].vector; +#endif + rv = request_irq(vector, xdma_channel_irq, 0, xdev->mod_name, + engine); + if (rv) { + pr_info("requesti irq#%d failed %d, engine %s.\n", + vector, rv, engine->name); + return rv; + } + pr_info("engine %s, irq#%d.\n", engine->name, vector); + engine->msix_irq_line = vector; + } + + return 0; +} + +static void irq_msix_user_teardown(struct xdma_dev *xdev) +{ + int i; + int j = xdev->h2c_channel_max + xdev->c2h_channel_max; + + BUG_ON(!xdev); + + if (!xdev->msix_enabled) + return; + + prog_irq_msix_user(xdev, 1); + + for (i = 0; i < xdev->user_max; i++, j++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + u32 vector = pci_irq_vector(xdev->pdev, j); +#else + u32 vector = xdev->entry[j].vector; +#endif + dbg_init("user %d, releasing IRQ#%d\n", i, vector); + free_irq(vector, &xdev->user_irq[i]); + } +} + +static int irq_msix_user_setup(struct xdma_dev *xdev) +{ + int i; + int j = xdev->h2c_channel_max + xdev->c2h_channel_max; + int rv = 0; + + /* vectors set in probe_scan_for_msi() */ + for (i = 0; i < xdev->user_max; i++, j++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + u32 vector = pci_irq_vector(xdev->pdev, j); +#else + u32 vector = xdev->entry[j].vector; +#endif + rv = request_irq(vector, xdma_user_irq, 0, xdev->mod_name, + &xdev->user_irq[i]); + if (rv) { + pr_info("user %d couldn't use IRQ#%d, %d\n", + i, vector, rv); + break; + } + pr_info("%d-USR-%d, IRQ#%d with 0x%p\n", xdev->idx, i, vector, + &xdev->user_irq[i]); + } + + /* If any errors occur, free IRQs that were successfully requested */ + if (rv) { + for (i--, j--; i >= 0; i--, j--) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + u32 vector = pci_irq_vector(xdev->pdev, j); +#else + u32 vector = xdev->entry[j].vector; +#endif + free_irq(vector, &xdev->user_irq[i]); + } + } + + return rv; +} + +static int irq_msi_setup(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + int rv; + + xdev->irq_line = (int)pdev->irq; + rv = request_irq(pdev->irq, xdma_isr, 0, xdev->mod_name, xdev); + if (rv) + dbg_init("Couldn't use IRQ#%d, %d\n", pdev->irq, rv); + else + dbg_init("Using IRQ#%d with 0x%p\n", pdev->irq, xdev); + + return rv; +} + +static int irq_legacy_setup(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + u32 w; + u8 val; + void *reg; + int rv; + + pci_read_config_byte(pdev, PCI_INTERRUPT_PIN, &val); + dbg_init("Legacy Interrupt register value = %d\n", val); + if (val > 1) { + val--; + w = (val<<24) | (val<<16) | (val<<8)| val; + /* Program IRQ Block Channel vactor and IRQ Block User vector + * with Legacy interrupt value */ + reg = xdev->bar[xdev->config_bar_idx] + 0x2080; // IRQ user + write_register(w, reg, 0x2080); + write_register(w, reg+0x4, 0x2084); + write_register(w, reg+0x8, 0x2088); + write_register(w, reg+0xC, 0x208C); + reg = xdev->bar[xdev->config_bar_idx] + 0x20A0; // IRQ Block + write_register(w, reg, 0x20A0); + write_register(w, reg+0x4, 0x20A4); + } + + xdev->irq_line = (int)pdev->irq; + rv = request_irq(pdev->irq, xdma_isr, IRQF_SHARED, xdev->mod_name, + xdev); + if (rv) + dbg_init("Couldn't use IRQ#%d, %d\n", pdev->irq, rv); + else + dbg_init("Using IRQ#%d with 0x%p\n", pdev->irq, xdev); + + return rv; +} + +static void irq_teardown(struct xdma_dev *xdev) +{ + if (xdev->msix_enabled) { + irq_msix_channel_teardown(xdev); + irq_msix_user_teardown(xdev); + } else if (xdev->irq_line != -1) { + dbg_init("Releasing IRQ#%d\n", xdev->irq_line); + free_irq(xdev->irq_line, xdev); + } +} + +static int irq_setup(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + pci_keep_intx_enabled(pdev); + + if (xdev->msix_enabled) { + int rv = irq_msix_channel_setup(xdev); + if (rv) + return rv; + rv = irq_msix_user_setup(xdev); + if (rv) + return rv; + prog_irq_msix_channel(xdev, 0); + prog_irq_msix_user(xdev, 0); + + return 0; + } else if (xdev->msi_enabled) + return irq_msi_setup(xdev, pdev); + + return irq_legacy_setup(xdev, pdev); +} + +#ifdef __LIBXDMA_DEBUG__ +static void dump_desc(struct xdma_desc *desc_virt) +{ + int j; + u32 *p = (u32 *)desc_virt; + static char * const field_name[] = { + "magic|extra_adjacent|control", "bytes", "src_addr_lo", + "src_addr_hi", "dst_addr_lo", "dst_addr_hi", "next_addr", + "next_addr_pad"}; + char *dummy; + + /* remove warning about unused variable when debug printing is off */ + dummy = field_name[0]; + + for (j = 0; j < 8; j += 1) { + pr_info("0x%08lx/0x%02lx: 0x%08x 0x%08x %s\n", + (uintptr_t)p, (uintptr_t)p & 15, (int)*p, + le32_to_cpu(*p), field_name[j]); + p++; + } + pr_info("\n"); +} + +static void transfer_dump(struct xdma_transfer *transfer) +{ + int i; + struct xdma_desc *desc_virt = transfer->desc_virt; + + pr_info("xfer 0x%p, state 0x%x, f 0x%x, dir %d, len %u, last %d.\n", + transfer, transfer->state, transfer->flags, transfer->dir, + transfer->len, transfer->last_in_request); + + pr_info("transfer 0x%p, desc %d, bus 0x%llx, adj %d.\n", + transfer, transfer->desc_num, (u64)transfer->desc_bus, + transfer->desc_adjacent); + for (i = 0; i < transfer->desc_num; i += 1) + dump_desc(desc_virt + i); +} +#endif /* __LIBXDMA_DEBUG__ */ + +/* xdma_desc_alloc() - Allocate cache-coherent array of N descriptors. + * + * Allocates an array of 'number' descriptors in contiguous PCI bus addressable + * memory. Chains the descriptors as a singly-linked list; the descriptor's + * next * pointer specifies the bus address of the next descriptor. + * + * + * @dev Pointer to pci_dev + * @number Number of descriptors to be allocated + * @desc_bus_p Pointer where to store the first descriptor bus address + * + * @return Virtual address of the first descriptor + * + */ +static void transfer_desc_init(struct xdma_transfer *transfer, int count) +{ + struct xdma_desc *desc_virt = transfer->desc_virt; + dma_addr_t desc_bus = transfer->desc_bus; + int i; + int adj = count - 1; + int extra_adj; + u32 temp_control; + + BUG_ON(count > XDMA_TRANSFER_MAX_DESC); + + /* create singly-linked list for SG DMA controller */ + for (i = 0; i < count - 1; i++) { + /* increment bus address to next in array */ + desc_bus += sizeof(struct xdma_desc); + + /* singly-linked list uses bus addresses */ + desc_virt[i].next_lo = cpu_to_le32(PCI_DMA_L(desc_bus)); + desc_virt[i].next_hi = cpu_to_le32(PCI_DMA_H(desc_bus)); + desc_virt[i].bytes = cpu_to_le32(0); + + /* any adjacent descriptors? */ + if (adj > 0) { + extra_adj = adj - 1; + if (extra_adj > MAX_EXTRA_ADJ) + extra_adj = MAX_EXTRA_ADJ; + + adj--; + } else { + extra_adj = 0; + } + + temp_control = DESC_MAGIC | (extra_adj << 8); + + desc_virt[i].control = cpu_to_le32(temp_control); + } + /* { i = number - 1 } */ + /* zero the last descriptor next pointer */ + desc_virt[i].next_lo = cpu_to_le32(0); + desc_virt[i].next_hi = cpu_to_le32(0); + desc_virt[i].bytes = cpu_to_le32(0); + + temp_control = DESC_MAGIC; + + desc_virt[i].control = cpu_to_le32(temp_control); +} + +/* xdma_desc_link() - Link two descriptors + * + * Link the first descriptor to a second descriptor, or terminate the first. + * + * @first first descriptor + * @second second descriptor, or NULL if first descriptor must be set as last. + * @second_bus bus address of second descriptor + */ +static void xdma_desc_link(struct xdma_desc *first, struct xdma_desc *second, + dma_addr_t second_bus) +{ + /* + * remember reserved control in first descriptor, but zero + * extra_adjacent! + */ + /* RTO - what's this about? Shouldn't it be 0x0000c0ffUL? */ + u32 control = le32_to_cpu(first->control) & 0x0000f0ffUL; + /* second descriptor given? */ + if (second) { + /* + * link last descriptor of 1st array to first descriptor of + * 2nd array + */ + first->next_lo = cpu_to_le32(PCI_DMA_L(second_bus)); + first->next_hi = cpu_to_le32(PCI_DMA_H(second_bus)); + WARN_ON(first->next_hi); + /* no second descriptor given */ + } else { + /* first descriptor is the last */ + first->next_lo = 0; + first->next_hi = 0; + } + /* merge magic, extra_adjacent and control field */ + control |= DESC_MAGIC; + + /* write bytes and next_num */ + first->control = cpu_to_le32(control); +} + +/* xdma_desc_adjacent -- Set how many descriptors are adjacent to this one */ +static void xdma_desc_adjacent(struct xdma_desc *desc, int next_adjacent) +{ + int extra_adj = 0; + /* remember reserved and control bits */ + u32 control = le32_to_cpu(desc->control) & 0x0000f0ffUL; + u32 max_adj_4k = 0; + + if (next_adjacent > 0) { + extra_adj = next_adjacent - 1; + if (extra_adj > MAX_EXTRA_ADJ){ + extra_adj = MAX_EXTRA_ADJ; + } + max_adj_4k = (0x1000 - ((le32_to_cpu(desc->next_lo))&0xFFF))/32 - 1; + if (extra_adj>max_adj_4k) { + extra_adj = max_adj_4k; + } + if(extra_adj<0){ + printk("Warning: extra_adj<0, converting it to 0\n"); + extra_adj = 0; + } + } + /* merge adjacent and control field */ + control |= 0xAD4B0000UL | (extra_adj << 8); + /* write control and next_adjacent */ + desc->control = cpu_to_le32(control); +} + +/* xdma_desc_control -- Set complete control field of a descriptor. */ +static void xdma_desc_control_set(struct xdma_desc *first, u32 control_field) +{ + /* remember magic and adjacent number */ + u32 control = le32_to_cpu(first->control) & ~(LS_BYTE_MASK); + + BUG_ON(control_field & ~(LS_BYTE_MASK)); + /* merge adjacent and control field */ + control |= control_field; + /* write control and next_adjacent */ + first->control = cpu_to_le32(control); +} + +/* xdma_desc_clear -- Clear bits in control field of a descriptor. */ +static void xdma_desc_control_clear(struct xdma_desc *first, u32 clear_mask) +{ + /* remember magic and adjacent number */ + u32 control = le32_to_cpu(first->control); + + BUG_ON(clear_mask & ~(LS_BYTE_MASK)); + + /* merge adjacent and control field */ + control &= (~clear_mask); + /* write control and next_adjacent */ + first->control = cpu_to_le32(control); +} + +/* xdma_desc_done - recycle cache-coherent linked list of descriptors. + * + * @dev Pointer to pci_dev + * @number Number of descriptors to be allocated + * @desc_virt Pointer to (i.e. virtual address of) first descriptor in list + * @desc_bus Bus address of first descriptor in list + */ +static inline void xdma_desc_done(struct xdma_desc *desc_virt) +{ + memset(desc_virt, 0, XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc)); +} + +/* xdma_desc() - Fill a descriptor with the transfer details + * + * @desc pointer to descriptor to be filled + * @addr root complex address + * @ep_addr end point address + * @len number of bytes, must be a (non-negative) multiple of 4. + * @dir, dma direction + * is the end point address. If zero, vice versa. + * + * Does not modify the next pointer + */ +static void xdma_desc_set(struct xdma_desc *desc, dma_addr_t rc_bus_addr, + u64 ep_addr, int len, int dir) +{ + /* transfer length */ + desc->bytes = cpu_to_le32(len); + if (dir == DMA_TO_DEVICE) { + /* read from root complex memory (source address) */ + desc->src_addr_lo = cpu_to_le32(PCI_DMA_L(rc_bus_addr)); + desc->src_addr_hi = cpu_to_le32(PCI_DMA_H(rc_bus_addr)); + /* write to end point address (destination address) */ + desc->dst_addr_lo = cpu_to_le32(PCI_DMA_L(ep_addr)); + desc->dst_addr_hi = cpu_to_le32(PCI_DMA_H(ep_addr)); + } else { + /* read from end point address (source address) */ + desc->src_addr_lo = cpu_to_le32(PCI_DMA_L(ep_addr)); + desc->src_addr_hi = cpu_to_le32(PCI_DMA_H(ep_addr)); + /* write to root complex memory (destination address) */ + desc->dst_addr_lo = cpu_to_le32(PCI_DMA_L(rc_bus_addr)); + desc->dst_addr_hi = cpu_to_le32(PCI_DMA_H(rc_bus_addr)); + } +} + +/* + * should hold the engine->lock; + */ +static void transfer_abort(struct xdma_engine *engine, + struct xdma_transfer *transfer) +{ + struct xdma_transfer *head; + + BUG_ON(!engine); + BUG_ON(!transfer); + BUG_ON(transfer->desc_num == 0); + + pr_info("abort transfer 0x%p, desc %d, engine desc queued %d.\n", + transfer, transfer->desc_num, engine->desc_dequeued); + + head = list_entry(engine->transfer_list.next, struct xdma_transfer, + entry); + if (head == transfer) + list_del(engine->transfer_list.next); + else + pr_info("engine %s, transfer 0x%p NOT found, 0x%p.\n", + engine->name, transfer, head); + + if (transfer->state == TRANSFER_STATE_SUBMITTED) + transfer->state = TRANSFER_STATE_ABORTED; +} + +/* transfer_queue() - Queue a DMA transfer on the engine + * + * @engine DMA engine doing the transfer + * @transfer DMA transfer submitted to the engine + * + * Takes and releases the engine spinlock + */ +static int transfer_queue(struct xdma_engine *engine, + struct xdma_transfer *transfer) +{ + int rv = 0; + struct xdma_transfer *transfer_started; + struct xdma_dev *xdev; + unsigned long flags; + + BUG_ON(!engine); + BUG_ON(!engine->xdev); + BUG_ON(!transfer); + BUG_ON(transfer->desc_num == 0); + dbg_tfr("transfer_queue(transfer=0x%p).\n", transfer); + + xdev = engine->xdev; + if (xdma_device_flag_check(xdev, XDEV_FLAG_OFFLINE)) { + pr_info("dev 0x%p offline, transfer 0x%p not queued.\n", + xdev, transfer); + return -EBUSY; + } + + /* lock the engine state */ + spin_lock_irqsave(&engine->lock, flags); + + engine->prev_cpu = get_cpu(); + put_cpu(); + + /* engine is being shutdown; do not accept new transfers */ + if (engine->shutdown & ENGINE_SHUTDOWN_REQUEST) { + pr_info("engine %s offline, transfer 0x%p not queued.\n", + engine->name, transfer); + rv = -EBUSY; + goto shutdown; + } + + /* mark the transfer as submitted */ + transfer->state = TRANSFER_STATE_SUBMITTED; + /* add transfer to the tail of the engine transfer queue */ + list_add_tail(&transfer->entry, &engine->transfer_list); + + /* engine is idle? */ + if (!engine->running) { + /* start engine */ + dbg_tfr("transfer_queue(): starting %s engine.\n", + engine->name); + transfer_started = engine_start(engine); + dbg_tfr("transfer=0x%p started %s engine with transfer 0x%p.\n", + transfer, engine->name, transfer_started); + } else { + dbg_tfr("transfer=0x%p queued, with %s engine running.\n", + transfer, engine->name); + } + +shutdown: + /* unlock the engine state */ + dbg_tfr("engine->running = %d\n", engine->running); + spin_unlock_irqrestore(&engine->lock, flags); + return rv; +} + +static void engine_alignments(struct xdma_engine *engine) +{ + u32 w; + u32 align_bytes; + u32 granularity_bytes; + u32 address_bits; + + w = read_register(&engine->regs->alignments); + dbg_init("engine %p name %s alignments=0x%08x\n", engine, + engine->name, (int)w); + + /* RTO - add some macros to extract these fields */ + align_bytes = (w & 0x00ff0000U) >> 16; + granularity_bytes = (w & 0x0000ff00U) >> 8; + address_bits = (w & 0x000000ffU); + + dbg_init("align_bytes = %d\n", align_bytes); + dbg_init("granularity_bytes = %d\n", granularity_bytes); + dbg_init("address_bits = %d\n", address_bits); + + if (w) { + engine->addr_align = align_bytes; + engine->len_granularity = granularity_bytes; + engine->addr_bits = address_bits; + } else { + /* Some default values if alignments are unspecified */ + engine->addr_align = 1; + engine->len_granularity = 1; + engine->addr_bits = 64; + } +} + +static void engine_free_resource(struct xdma_engine *engine) +{ + struct xdma_dev *xdev = engine->xdev; + + /* Release memory use for descriptor writebacks */ + if (engine->poll_mode_addr_virt) { + dbg_sg("Releasing memory for descriptor writeback\n"); + dma_free_coherent(&xdev->pdev->dev, + sizeof(struct xdma_poll_wb), + engine->poll_mode_addr_virt, + engine->poll_mode_bus); + dbg_sg("Released memory for descriptor writeback\n"); + engine->poll_mode_addr_virt = NULL; + } + + if (engine->desc) { + dbg_init("device %s, engine %s pre-alloc desc 0x%p,0x%llx.\n", + dev_name(&xdev->pdev->dev), engine->name, + engine->desc, engine->desc_bus); + dma_free_coherent(&xdev->pdev->dev, + XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc), + engine->desc, engine->desc_bus); + engine->desc = NULL; + } + + if (engine->cyclic_result) { + dma_free_coherent(&xdev->pdev->dev, + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result), + engine->cyclic_result, engine->cyclic_result_bus); + engine->cyclic_result = NULL; + } +} + +static void engine_destroy(struct xdma_dev *xdev, struct xdma_engine *engine) +{ + BUG_ON(!xdev); + BUG_ON(!engine); + + dbg_sg("Shutting down engine %s%d", engine->name, engine->channel); + + /* Disable interrupts to stop processing new events during shutdown */ + write_register(0x0, &engine->regs->interrupt_enable_mask, + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); + + if (enable_credit_mp && engine->streaming && + engine->dir == DMA_FROM_DEVICE) { + u32 reg_value = (0x1 << engine->channel) << 16; + struct sgdma_common_regs *reg = (struct sgdma_common_regs *) + (xdev->bar[xdev->config_bar_idx] + + (0x6*TARGET_SPACING)); + write_register(reg_value, ®->credit_mode_enable_w1c, 0); + } + + /* Release memory use for descriptor writebacks */ + engine_free_resource(engine); + + memset(engine, 0, sizeof(struct xdma_engine)); + /* Decrement the number of engines available */ + xdev->engines_num--; +} + +/** + *engine_cyclic_stop() - stop a cyclic transfer running on an SG DMA engine + * + *engine->lock must be taken + */ +struct xdma_transfer *engine_cyclic_stop(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer = 0; + + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) { + /* pick first transfer on the queue (was submitted to engine) */ + transfer = list_entry(engine->transfer_list.next, + struct xdma_transfer, entry); + BUG_ON(!transfer); + + xdma_engine_stop(engine); + + if (transfer->cyclic) { + if (engine->xdma_perf) + dbg_perf("Stopping perf transfer on %s\n", + engine->name); + else + dbg_perf("Stopping cyclic transfer on %s\n", + engine->name); + /* make sure the handler sees correct transfer state */ + transfer->cyclic = 1; + /* + * set STOP flag and interrupt on completion, on the + * last descriptor + */ + xdma_desc_control_set( + transfer->desc_virt + transfer->desc_num - 1, + XDMA_DESC_COMPLETED | XDMA_DESC_STOPPED); + } else { + dbg_sg("(engine=%p) running transfer is not cyclic\n", + engine); + } + } else { + dbg_sg("(engine=%p) found not running transfer.\n", engine); + } + return transfer; +} +EXPORT_SYMBOL_GPL(engine_cyclic_stop); + +static int engine_writeback_setup(struct xdma_engine *engine) +{ + u32 w; + struct xdma_dev *xdev; + struct xdma_poll_wb *writeback; + + BUG_ON(!engine); + xdev = engine->xdev; + BUG_ON(!xdev); + + /* + * RTO - doing the allocation per engine is wasteful since a full page + * is allocated each time - better to allocate one page for the whole + * device during probe() and set per-engine offsets here + */ + writeback = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + writeback->completed_desc_count = 0; + + dbg_init("Setting writeback location to 0x%llx for engine %p", + engine->poll_mode_bus, engine); + w = cpu_to_le32(PCI_DMA_L(engine->poll_mode_bus)); + write_register(w, &engine->regs->poll_mode_wb_lo, + (unsigned long)(&engine->regs->poll_mode_wb_lo) - + (unsigned long)(&engine->regs)); + w = cpu_to_le32(PCI_DMA_H(engine->poll_mode_bus)); + write_register(w, &engine->regs->poll_mode_wb_hi, + (unsigned long)(&engine->regs->poll_mode_wb_hi) - + (unsigned long)(&engine->regs)); + + return 0; +} + + +/* engine_create() - Create an SG DMA engine bookkeeping data structure + * + * An SG DMA engine consists of the resources for a single-direction transfer + * queue; the SG DMA hardware, the software queue and interrupt handling. + * + * @dev Pointer to pci_dev + * @offset byte address offset in BAR[xdev->config_bar_idx] resource for the + * SG DMA * controller registers. + * @dir: DMA_TO/FROM_DEVICE + * @streaming Whether the engine is attached to AXI ST (rather than MM) + */ +static int engine_init_regs(struct xdma_engine *engine) +{ + u32 reg_value; + int rv = 0; + + write_register(XDMA_CTRL_NON_INCR_ADDR, &engine->regs->control_w1c, + (unsigned long)(&engine->regs->control_w1c) - + (unsigned long)(&engine->regs)); + + engine_alignments(engine); + + /* Configure error interrupts by default */ + reg_value = XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + reg_value |= XDMA_CTRL_IE_MAGIC_STOPPED; + reg_value |= XDMA_CTRL_IE_MAGIC_STOPPED; + reg_value |= XDMA_CTRL_IE_READ_ERROR; + reg_value |= XDMA_CTRL_IE_DESC_ERROR; + + /* if using polled mode, configure writeback address */ + if (poll_mode) { + rv = engine_writeback_setup(engine); + if (rv) { + dbg_init("%s descr writeback setup failed.\n", + engine->name); + goto fail_wb; + } + } else { + /* enable the relevant completion interrupts */ + reg_value |= XDMA_CTRL_IE_DESC_STOPPED; + reg_value |= XDMA_CTRL_IE_DESC_COMPLETED; + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) + reg_value |= XDMA_CTRL_IE_IDLE_STOPPED; + } + + /* Apply engine configurations */ + write_register(reg_value, &engine->regs->interrupt_enable_mask, + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); + + engine->interrupt_enable_mask_value = reg_value; + + /* only enable credit mode for AXI-ST C2H */ + if (enable_credit_mp && engine->streaming && + engine->dir == DMA_FROM_DEVICE) { + + struct xdma_dev *xdev = engine->xdev; + u32 reg_value = (0x1 << engine->channel) << 16; + struct sgdma_common_regs *reg = (struct sgdma_common_regs *) + (xdev->bar[xdev->config_bar_idx] + + (0x6*TARGET_SPACING)); + + write_register(reg_value, ®->credit_mode_enable_w1s, 0); + } + + return 0; + +fail_wb: + return rv; +} + +static int engine_alloc_resource(struct xdma_engine *engine) +{ + struct xdma_dev *xdev = engine->xdev; + + engine->desc = dma_alloc_coherent(&xdev->pdev->dev, + XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc), + &engine->desc_bus, GFP_KERNEL); + if (!engine->desc) { + pr_warn("dev %s, %s pre-alloc desc OOM.\n", + dev_name(&xdev->pdev->dev), engine->name); + goto err_out; + } + + if (poll_mode) { + engine->poll_mode_addr_virt = dma_alloc_coherent( + &xdev->pdev->dev, + sizeof(struct xdma_poll_wb), + &engine->poll_mode_bus, GFP_KERNEL); + if (!engine->poll_mode_addr_virt) { + pr_warn("%s, %s poll pre-alloc writeback OOM.\n", + dev_name(&xdev->pdev->dev), engine->name); + goto err_out; + } + } + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { + engine->cyclic_result = dma_alloc_coherent(&xdev->pdev->dev, + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result), + &engine->cyclic_result_bus, GFP_KERNEL); + + if (!engine->cyclic_result) { + pr_warn("%s, %s pre-alloc result OOM.\n", + dev_name(&xdev->pdev->dev), engine->name); + goto err_out; + } + } + + return 0; + +err_out: + engine_free_resource(engine); + return -ENOMEM; +} + +static int engine_init(struct xdma_engine *engine, struct xdma_dev *xdev, + int offset, enum dma_data_direction dir, int channel) +{ + int rv; + u32 val; + + dbg_init("channel %d, offset 0x%x, dir %d.\n", channel, offset, dir); + + /* set magic */ + engine->magic = MAGIC_ENGINE; + + engine->channel = channel; + + /* engine interrupt request bit */ + engine->irq_bitmask = (1 << XDMA_ENG_IRQ_NUM) - 1; + engine->irq_bitmask <<= (xdev->engines_num * XDMA_ENG_IRQ_NUM); + engine->bypass_offset = xdev->engines_num * BYPASS_MODE_SPACING; + + /* parent */ + engine->xdev = xdev; + /* register address */ + engine->regs = (xdev->bar[xdev->config_bar_idx] + offset); + engine->sgdma_regs = xdev->bar[xdev->config_bar_idx] + offset + + SGDMA_OFFSET_FROM_CHANNEL; + val = read_register(&engine->regs->identifier); + if (val & 0x8000U) + engine->streaming = 1; + + /* remember SG DMA direction */ + engine->dir = dir; + sprintf(engine->name, "%d-%s%d-%s", xdev->idx, + (dir == DMA_TO_DEVICE) ? "H2C" : "C2H", channel, + engine->streaming ? "ST" : "MM"); + + dbg_init("engine %p name %s irq_bitmask=0x%08x\n", engine, engine->name, + (int)engine->irq_bitmask); + + /* initialize the deferred work for transfer completion */ + INIT_WORK(&engine->work, engine_service_work); + + if (dir == DMA_TO_DEVICE) + xdev->mask_irq_h2c |= engine->irq_bitmask; + else + xdev->mask_irq_c2h |= engine->irq_bitmask; + xdev->engines_num++; + + rv = engine_alloc_resource(engine); + if (rv) + return rv; + + rv = engine_init_regs(engine); + if (rv) + return rv; + + return 0; +} + +/* transfer_destroy() - free transfer */ +static void transfer_destroy(struct xdma_dev *xdev, struct xdma_transfer *xfer) +{ + /* free descriptors */ + xdma_desc_done(xfer->desc_virt); + + if (xfer->last_in_request && (xfer->flags & XFER_FLAG_NEED_UNMAP)) { + struct sg_table *sgt = xfer->sgt; + + if (sgt->nents) { + pci_unmap_sg(xdev->pdev, sgt->sgl, sgt->nents, + xfer->dir); + sgt->nents = 0; + } + } +} + +static int transfer_build(struct xdma_engine *engine, + struct xdma_request_cb *req, unsigned int desc_max) +{ + struct xdma_transfer *xfer = &req->xfer; + struct sw_desc *sdesc = &(req->sdesc[req->sw_desc_idx]); + int i = 0; + int j = 0; + + for (; i < desc_max; i++, j++, sdesc++) { + dbg_desc("sw desc %d/%u: 0x%llx, 0x%x, ep 0x%llx.\n", + i + req->sw_desc_idx, req->sw_desc_cnt, + sdesc->addr, sdesc->len, req->ep_addr); + + /* fill in descriptor entry j with transfer details */ + xdma_desc_set(xfer->desc_virt + j, sdesc->addr, req->ep_addr, + sdesc->len, xfer->dir); + xfer->len += sdesc->len; + + /* for non-inc-add mode don't increment ep_addr */ + if (!engine->non_incr_addr) + req->ep_addr += sdesc->len; + } + req->sw_desc_idx += desc_max; + return 0; +} + +static int transfer_init(struct xdma_engine *engine, struct xdma_request_cb *req) +{ + struct xdma_transfer *xfer = &req->xfer; + unsigned int desc_max = min_t(unsigned int, + req->sw_desc_cnt - req->sw_desc_idx, + XDMA_TRANSFER_MAX_DESC); + int i = 0; + int last = 0; + u32 control; + + memset(xfer, 0, sizeof(*xfer)); + + /* initialize wait queue */ + init_waitqueue_head(&xfer->wq); + + /* remember direction of transfer */ + xfer->dir = engine->dir; + + xfer->desc_virt = engine->desc; + xfer->desc_bus = engine->desc_bus; + + transfer_desc_init(xfer, desc_max); + + dbg_sg("transfer->desc_bus = 0x%llx.\n", (u64)xfer->desc_bus); + + transfer_build(engine, req, desc_max); + + /* terminate last descriptor */ + last = desc_max - 1; + xdma_desc_link(xfer->desc_virt + last, 0, 0); + /* stop engine, EOP for AXI ST, req IRQ on last descriptor */ + control = XDMA_DESC_STOPPED; + control |= XDMA_DESC_EOP; + control |= XDMA_DESC_COMPLETED; + xdma_desc_control_set(xfer->desc_virt + last, control); + + xfer->desc_num = xfer->desc_adjacent = desc_max; + + dbg_sg("transfer 0x%p has %d descriptors\n", xfer, xfer->desc_num); + /* fill in adjacent numbers */ + for (i = 0; i < xfer->desc_num; i++) + xdma_desc_adjacent(xfer->desc_virt + i, xfer->desc_num - i - 1); + + return 0; +} + +#ifdef __LIBXDMA_DEBUG__ +static void sgt_dump(struct sg_table *sgt) +{ + int i; + struct scatterlist *sg = sgt->sgl; + + pr_info("sgt 0x%p, sgl 0x%p, nents %u/%u.\n", + sgt, sgt->sgl, sgt->nents, sgt->orig_nents); + + for (i = 0; i < sgt->orig_nents; i++, sg = sg_next(sg)) + pr_info("%d, 0x%p, pg 0x%p,%u+%u, dma 0x%llx,%u.\n", + i, sg, sg_page(sg), sg->offset, sg->length, + sg_dma_address(sg), sg_dma_len(sg)); +} + +static void xdma_request_cb_dump(struct xdma_request_cb *req) +{ + int i; + + pr_info("request 0x%p, total %u, ep 0x%llx, sw_desc %u, sgt 0x%p.\n", + req, req->total_len, req->ep_addr, req->sw_desc_cnt, req->sgt); + sgt_dump(req->sgt); + for (i = 0; i < req->sw_desc_cnt; i++) + pr_info("%d/%u, 0x%llx, %u.\n", + i, req->sw_desc_cnt, req->sdesc[i].addr, + req->sdesc[i].len); +} +#endif + +static void xdma_request_free(struct xdma_request_cb *req) +{ + if (((unsigned long)req) >= VMALLOC_START && + ((unsigned long)req) < VMALLOC_END) + vfree(req); + else + kfree(req); +} + +static struct xdma_request_cb * xdma_request_alloc(unsigned int sdesc_nr) +{ + struct xdma_request_cb *req; + unsigned int size = sizeof(struct xdma_request_cb) + + sdesc_nr * sizeof(struct sw_desc); + + req = kzalloc(size, GFP_KERNEL); + if (!req) { + req = vmalloc(size); + if (req) + memset(req, 0, size); + } + if (!req) { + pr_info("OOM, %u sw_desc, %u.\n", sdesc_nr, size); + return NULL; + } + + return req; +} + +static struct xdma_request_cb * xdma_init_request(struct sg_table *sgt, + u64 ep_addr) +{ + struct xdma_request_cb *req; + struct scatterlist *sg = sgt->sgl; + int max = sgt->nents; + int extra = 0; + int i, j = 0; + + for (i = 0; i < max; i++, sg = sg_next(sg)) { + unsigned int len = sg_dma_len(sg); + + if (unlikely(len > desc_blen_max)) + extra += (len + desc_blen_max - 1) / desc_blen_max; + } + +//pr_info("ep 0x%llx, desc %u+%u.\n", ep_addr, max, extra); + + max += extra; + req = xdma_request_alloc(max); + if (!req) + return NULL; + + req->sgt = sgt; + req->ep_addr = ep_addr; + + for (i = 0, sg = sgt->sgl; i < sgt->nents; i++, sg = sg_next(sg)) { + unsigned int tlen = sg_dma_len(sg); + dma_addr_t addr = sg_dma_address(sg); + + req->total_len += tlen; + while (tlen) { + req->sdesc[j].addr = addr; + if (tlen > desc_blen_max) { + req->sdesc[j].len = desc_blen_max; + addr += desc_blen_max; + tlen -= desc_blen_max; + } else { + req->sdesc[j].len = tlen; + tlen = 0; + } + j++; + } + } + BUG_ON(j > max); + + req->sw_desc_cnt = j; +#ifdef __LIBXDMA_DEBUG__ + xdma_request_cb_dump(req); +#endif + return req; +} + +ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, + struct sg_table *sgt, bool dma_mapped, int timeout_ms) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + struct xdma_engine *engine; + int rv = 0; + ssize_t done = 0; + struct scatterlist *sg = sgt->sgl; + int nents; + enum dma_data_direction dir = write ? DMA_TO_DEVICE : DMA_FROM_DEVICE; + struct xdma_request_cb *req = NULL; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + if (write == 1) { + if (channel >= xdev->h2c_channel_max) { + pr_warn("H2C channel %d >= %d.\n", + channel, xdev->h2c_channel_max); + return -EINVAL; + } + engine = &xdev->engine_h2c[channel]; + } else if (write == 0) { + if (channel >= xdev->c2h_channel_max) { + pr_warn("C2H channel %d >= %d.\n", + channel, xdev->c2h_channel_max); + return -EINVAL; + } + engine = &xdev->engine_c2h[channel]; + } else { + pr_warn("write %d, exp. 0|1.\n", write); + return -EINVAL; + } + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + xdev = engine->xdev; + if (xdma_device_flag_check(xdev, XDEV_FLAG_OFFLINE)) { + pr_info("xdev 0x%p, offline.\n", xdev); + return -EBUSY; + } + + /* check the direction */ + if (engine->dir != dir) { + pr_info("0x%p, %s, %d, W %d, 0x%x/0x%x mismatch.\n", + engine, engine->name, channel, write, engine->dir, dir); + return -EINVAL; + } + + if (!dma_mapped) { + nents = pci_map_sg(xdev->pdev, sg, sgt->orig_nents, dir); + if (!nents) { + pr_info("map sgl failed, sgt 0x%p.\n", sgt); + return -EIO; + } + sgt->nents = nents; + } else { + BUG_ON(!sgt->nents); + } + + req = xdma_init_request(sgt, ep_addr); + if (!req) { + rv = -ENOMEM; + goto unmap_sgl; + } + + dbg_tfr("%s, len %u sg cnt %u.\n", + engine->name, req->total_len, req->sw_desc_cnt); + + sg = sgt->sgl; + nents = req->sw_desc_cnt; + while (nents) { + unsigned long flags; + struct xdma_transfer *xfer; + + /* one transfer at a time */ + spin_lock(&engine->desc_lock); + + /* build transfer */ + rv = transfer_init(engine, req); + if (rv < 0) { + spin_unlock(&engine->desc_lock); + goto unmap_sgl; + } + xfer = &req->xfer; + + if (!dma_mapped) + xfer->flags = XFER_FLAG_NEED_UNMAP; + + /* last transfer for the given request? */ + nents -= xfer->desc_num; + if (!nents) { + xfer->last_in_request = 1; + xfer->sgt = sgt; + } + + dbg_tfr("xfer, %u, ep 0x%llx, done %lu, sg %u/%u.\n", + xfer->len, req->ep_addr, done, req->sw_desc_idx, + req->sw_desc_cnt); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); +#endif + + rv = transfer_queue(engine, xfer); + if (rv < 0) { + spin_unlock(&engine->desc_lock); + pr_info("unable to submit %s, %d.\n", engine->name, rv); + goto unmap_sgl; + } + + /* + * When polling, determine how many descriptors have been queued * on the engine to determine the writeback value expected + */ + if (poll_mode) { + unsigned int desc_count; + + spin_lock_irqsave(&engine->lock, flags); + desc_count = xfer->desc_num; + spin_unlock_irqrestore(&engine->lock, flags); + + dbg_tfr("%s poll desc_count=%d\n", + engine->name, desc_count); + rv = engine_service_poll(engine, desc_count); + + } else { + rv = wait_event_interruptible_timeout(xfer->wq, + (xfer->state != TRANSFER_STATE_SUBMITTED), + msecs_to_jiffies(timeout_ms)); + } + + spin_lock_irqsave(&engine->lock, flags); + + switch(xfer->state) { + case TRANSFER_STATE_COMPLETED: + spin_unlock_irqrestore(&engine->lock, flags); + + dbg_tfr("transfer %p, %u, ep 0x%llx compl, +%lu.\n", + xfer, xfer->len, req->ep_addr - xfer->len, done); + done += xfer->len; + rv = 0; + break; + case TRANSFER_STATE_FAILED: + pr_info("xfer 0x%p,%u, failed, ep 0x%llx.\n", + xfer, xfer->len, req->ep_addr - xfer->len); + spin_unlock_irqrestore(&engine->lock, flags); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); + sgt_dump(sgt); +#endif + rv = -EIO; + break; + default: + /* transfer can still be in-flight */ + pr_info("xfer 0x%p,%u, s 0x%x timed out, ep 0x%llx.\n", + xfer, xfer->len, xfer->state, req->ep_addr); + engine_status_read(engine, 0, 1); + //engine_status_dump(engine); + transfer_abort(engine, xfer); + + xdma_engine_stop(engine); + spin_unlock_irqrestore(&engine->lock, flags); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); + sgt_dump(sgt); +#endif + rv = -ERESTARTSYS; + break; + } + + transfer_destroy(xdev, xfer); + spin_unlock(&engine->desc_lock); + + if (rv < 0) + goto unmap_sgl; + } /* while (sg) */ + +unmap_sgl: + if (!dma_mapped && sgt->nents) { + pci_unmap_sg(xdev->pdev, sgt->sgl, sgt->orig_nents, dir); + sgt->nents = 0; + } + + if (req) + xdma_request_free(req); + + if (rv < 0) + return rv; + + return done; +} +EXPORT_SYMBOL_GPL(xdma_xfer_submit); + +int xdma_performance_submit(struct xdma_dev *xdev, struct xdma_engine *engine) +{ + u8 *buffer_virt; + u32 max_consistent_size = 128 * 32 * 1024; /* 1024 pages, 4MB */ + dma_addr_t buffer_bus; /* bus address */ + struct xdma_transfer *transfer; + u64 ep_addr = 0; + int num_desc_in_a_loop = 128; + int size_in_desc = engine->xdma_perf->transfer_size; + int size = size_in_desc * num_desc_in_a_loop; + int i; + + BUG_ON(size_in_desc > max_consistent_size); + + if (size > max_consistent_size) { + size = max_consistent_size; + num_desc_in_a_loop = size / size_in_desc; + } + + buffer_virt = dma_alloc_coherent(&xdev->pdev->dev, size, + &buffer_bus, GFP_KERNEL); + + /* allocate transfer data structure */ + transfer = kzalloc(sizeof(struct xdma_transfer), GFP_KERNEL); + BUG_ON(!transfer); + + /* 0 = write engine (to_dev=0) , 1 = read engine (to_dev=1) */ + transfer->dir = engine->dir; + /* set number of descriptors */ + transfer->desc_num = num_desc_in_a_loop; + + /* allocate descriptor list */ + if (!engine->desc) { + engine->desc = dma_alloc_coherent(&xdev->pdev->dev, + num_desc_in_a_loop * sizeof(struct xdma_desc), + &engine->desc_bus, GFP_KERNEL); + BUG_ON(!engine->desc); + dbg_init("device %s, engine %s pre-alloc desc 0x%p,0x%llx.\n", + dev_name(&xdev->pdev->dev), engine->name, + engine->desc, engine->desc_bus); + } + transfer->desc_virt = engine->desc; + transfer->desc_bus = engine->desc_bus; + + transfer_desc_init(transfer, transfer->desc_num); + + dbg_sg("transfer->desc_bus = 0x%llx.\n", (u64)transfer->desc_bus); + + for (i = 0; i < transfer->desc_num; i++) { + struct xdma_desc *desc = transfer->desc_virt + i; + dma_addr_t rc_bus_addr = buffer_bus + size_in_desc * i; + + /* fill in descriptor entry with transfer details */ + xdma_desc_set(desc, rc_bus_addr, ep_addr, size_in_desc, + engine->dir); + } + + /* stop engine and request interrupt on last descriptor */ + xdma_desc_control_set(transfer->desc_virt, 0); + /* create a linked loop */ + xdma_desc_link(transfer->desc_virt + transfer->desc_num - 1, + transfer->desc_virt, transfer->desc_bus); + + transfer->cyclic = 1; + + /* initialize wait queue */ + init_waitqueue_head(&transfer->wq); + + //printk("=== Descriptor print for PERF \n"); + //transfer_dump(transfer); + + dbg_perf("Queueing XDMA I/O %s request for performance measurement.\n", + engine->dir ? "write (to dev)" : "read (from dev)"); + transfer_queue(engine, transfer); + return 0; + +} +EXPORT_SYMBOL_GPL(xdma_performance_submit); + +static struct xdma_dev *alloc_dev_instance(struct pci_dev *pdev) +{ + int i; + struct xdma_dev *xdev; + struct xdma_engine *engine; + + BUG_ON(!pdev); + + /* allocate zeroed device book keeping structure */ + xdev = kzalloc(sizeof(struct xdma_dev), GFP_KERNEL); + if (!xdev) { + pr_info("OOM, xdma_dev.\n"); + return NULL; + } + spin_lock_init(&xdev->lock); + + xdev->magic = MAGIC_DEVICE; + xdev->config_bar_idx = -1; + xdev->user_bar_idx = -1; + xdev->bypass_bar_idx = -1; + xdev->irq_line = -1; + + /* create a driver to device reference */ + xdev->pdev = pdev; + dbg_init("xdev = 0x%p\n", xdev); + + /* Set up data user IRQ data structures */ + for (i = 0; i < 16; i++) { + xdev->user_irq[i].xdev = xdev; + spin_lock_init(&xdev->user_irq[i].events_lock); + init_waitqueue_head(&xdev->user_irq[i].events_wq); + xdev->user_irq[i].handler = NULL; + xdev->user_irq[i].user_idx = i; /* 0 based */ + } + + engine = xdev->engine_h2c; + for (i = 0; i < XDMA_CHANNEL_NUM_MAX; i++, engine++) { + spin_lock_init(&engine->lock); + spin_lock_init(&engine->desc_lock); + INIT_LIST_HEAD(&engine->transfer_list); + init_waitqueue_head(&engine->shutdown_wq); + init_waitqueue_head(&engine->xdma_perf_wq); + } + + engine = xdev->engine_c2h; + for (i = 0; i < XDMA_CHANNEL_NUM_MAX; i++, engine++) { + spin_lock_init(&engine->lock); + spin_lock_init(&engine->desc_lock); + INIT_LIST_HEAD(&engine->transfer_list); + init_waitqueue_head(&engine->shutdown_wq); + init_waitqueue_head(&engine->xdma_perf_wq); + } + + return xdev; +} + +static int request_regions(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + int rv; + + BUG_ON(!xdev); + BUG_ON(!pdev); + + dbg_init("pci_request_regions()\n"); + rv = pci_request_regions(pdev, xdev->mod_name); + /* could not request all regions? */ + if (rv) { + dbg_init("pci_request_regions() = %d, device in use?\n", rv); + /* assume device is in use so do not disable it later */ + xdev->regions_in_use = 1; + } else { + xdev->got_regions = 1; + } + + return rv; +} + +static int set_dma_mask(struct pci_dev *pdev) +{ + BUG_ON(!pdev); + + dbg_init("sizeof(dma_addr_t) == %ld\n", sizeof(dma_addr_t)); + /* 64-bit addressing capability for XDMA? */ + if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + /* query for DMA transfer */ + /* @see Documentation/DMA-mapping.txt */ + dbg_init("pci_set_dma_mask()\n"); + /* use 64-bit DMA */ + dbg_init("Using a 64-bit DMA mask.\n"); + /* use 32-bit DMA for descriptors */ + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + /* use 64-bit DMA, 32-bit for consistent */ + } else if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) { + dbg_init("Could not set 64-bit DMA mask.\n"); + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + /* use 32-bit DMA */ + dbg_init("Using a 32-bit DMA mask.\n"); + } else { + dbg_init("No suitable DMA possible.\n"); + return -EINVAL; + } + + return 0; +} + +static u32 get_engine_channel_id(struct engine_regs *regs) +{ + u32 value; + + BUG_ON(!regs); + + value = read_register(®s->identifier); + + return (value & 0x00000f00U) >> 8; +} + +static u32 get_engine_id(struct engine_regs *regs) +{ + u32 value; + + BUG_ON(!regs); + + value = read_register(®s->identifier); + return (value & 0xffff0000U) >> 16; +} + +static void remove_engines(struct xdma_dev *xdev) +{ + struct xdma_engine *engine; + int i; + + BUG_ON(!xdev); + + /* iterate over channels */ + for (i = 0; i < xdev->h2c_channel_max; i++) { + engine = &xdev->engine_h2c[i]; + if (engine->magic == MAGIC_ENGINE) { + dbg_sg("Remove %s, %d", engine->name, i); + engine_destroy(xdev, engine); + dbg_sg("%s, %d removed", engine->name, i); + } + } + + for (i = 0; i < xdev->c2h_channel_max; i++) { + engine = &xdev->engine_c2h[i]; + if (engine->magic == MAGIC_ENGINE) { + dbg_sg("Remove %s, %d", engine->name, i); + engine_destroy(xdev, engine); + dbg_sg("%s, %d removed", engine->name, i); + } + } +} + +static int probe_for_engine(struct xdma_dev *xdev, enum dma_data_direction dir, + int channel) +{ + struct engine_regs *regs; + int offset = channel * CHANNEL_SPACING; + u32 engine_id; + u32 engine_id_expected; + u32 channel_id; + struct xdma_engine *engine; + int rv; + + /* register offset for the engine */ + /* read channels at 0x0000, write channels at 0x1000, + * channels at 0x100 interval */ + if (dir == DMA_TO_DEVICE) { + engine_id_expected = XDMA_ID_H2C; + engine = &xdev->engine_h2c[channel]; + } else { + offset += H2C_CHANNEL_OFFSET; + engine_id_expected = XDMA_ID_C2H; + engine = &xdev->engine_c2h[channel]; + } + + regs = xdev->bar[xdev->config_bar_idx] + offset; + engine_id = get_engine_id(regs); + channel_id = get_engine_channel_id(regs); + + if ((engine_id != engine_id_expected) || (channel_id != channel)) { + dbg_init("%s %d engine, reg off 0x%x, id mismatch 0x%x,0x%x," + "exp 0x%x,0x%x, SKIP.\n", + dir == DMA_TO_DEVICE ? "H2C" : "C2H", + channel, offset, engine_id, channel_id, + engine_id_expected, channel_id != channel); + return -EINVAL; + } + + dbg_init("found AXI %s %d engine, reg. off 0x%x, id 0x%x,0x%x.\n", + dir == DMA_TO_DEVICE ? "H2C" : "C2H", channel, + offset, engine_id, channel_id); + + /* allocate and initialize engine */ + rv = engine_init(engine, xdev, offset, dir, channel); + if (rv != 0) { + pr_info("failed to create AXI %s %d engine.\n", + dir == DMA_TO_DEVICE ? "H2C" : "C2H", + channel); + return rv; + } + + return 0; +} + +static int probe_engines(struct xdma_dev *xdev) +{ + int i; + int rv = 0; + + BUG_ON(!xdev); + + /* iterate over channels */ + for (i = 0; i < xdev->h2c_channel_max; i++) { + rv = probe_for_engine(xdev, DMA_TO_DEVICE, i); + if (rv) + break; + } + xdev->h2c_channel_max = i; + + for (i = 0; i < xdev->c2h_channel_max; i++) { + rv = probe_for_engine(xdev, DMA_FROM_DEVICE, i); + if (rv) + break; + } + xdev->c2h_channel_max = i; + + return 0; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0) +static void pci_enable_relaxed_ordering(struct pci_dev *pdev) +{ + pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN); +} +#else +static void pci_enable_relaxed_ordering(struct pci_dev *pdev) +{ + u16 v; + int pos; + + pos = pci_pcie_cap(pdev); + if (pos > 0) { + pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &v); + v |= PCI_EXP_DEVCTL_RELAX_EN; + pci_write_config_word(pdev, pos + PCI_EXP_DEVCTL, v); + } +} +#endif + +static void pci_check_extended_tag(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + u16 cap; + u32 v; + void *__iomem reg; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0) + pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &cap); +#else + int pos; + + pos = pci_pcie_cap(pdev); + if (pos > 0) + pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &cap); + else { + pr_info("pdev 0x%p, unable to access pcie cap.\n", pdev); + return; + } +#endif + + if ((cap & PCI_EXP_DEVCTL_EXT_TAG)) + return; + + /* extended tag not enabled */ + pr_info("0x%p EXT_TAG disabled.\n", pdev); + + if (xdev->config_bar_idx < 0) { + pr_info("pdev 0x%p, xdev 0x%p, config bar UNKNOWN.\n", + pdev, xdev); + return; + } + + reg = xdev->bar[xdev->config_bar_idx] + XDMA_OFS_CONFIG + 0x4C; + v = read_register(reg); + v = (v & 0xFF) | (((u32)32) << 8); + write_register(v, reg, XDMA_OFS_CONFIG + 0x4C); +} + +void *xdma_device_open(const char *mname, struct pci_dev *pdev, int *user_max, + int *h2c_channel_max, int *c2h_channel_max) +{ + struct xdma_dev *xdev = NULL; + int rv = 0; + + pr_info("%s device %s, 0x%p.\n", mname, dev_name(&pdev->dev), pdev); + + /* allocate zeroed device book keeping structure */ + xdev = alloc_dev_instance(pdev); + if (!xdev) + return NULL; + xdev->mod_name = mname; + xdev->user_max = *user_max; + xdev->h2c_channel_max = *h2c_channel_max; + xdev->c2h_channel_max = *c2h_channel_max; + + xdma_device_flag_set(xdev, XDEV_FLAG_OFFLINE); + xdev_list_add(xdev); + + if (xdev->user_max == 0 || xdev->user_max > MAX_USER_IRQ) + xdev->user_max = MAX_USER_IRQ; + if (xdev->h2c_channel_max == 0 || + xdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX) + xdev->h2c_channel_max = XDMA_CHANNEL_NUM_MAX; + if (xdev->c2h_channel_max == 0 || + xdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX) + xdev->c2h_channel_max = XDMA_CHANNEL_NUM_MAX; + + rv = pci_enable_device(pdev); + if (rv) { + dbg_init("pci_enable_device() failed, %d.\n", rv); + goto err_enable; + } + + /* keep INTx enabled */ + pci_check_intr_pend(pdev); + + /* enable relaxed ordering */ + pci_enable_relaxed_ordering(pdev); + + pci_check_extended_tag(xdev, pdev); + + /* force MRRS to be 512 */ + rv = pcie_set_readrq(pdev, 512); + if (rv) + pr_info("device %s, error set PCI_EXP_DEVCTL_READRQ: %d.\n", + dev_name(&pdev->dev), rv); + + /* enable bus master capability */ + pci_set_master(pdev); + + rv = request_regions(xdev, pdev); + if (rv) + goto err_regions; + + rv = map_bars(xdev, pdev); + if (rv) + goto err_map; + + rv = set_dma_mask(pdev); + if (rv) + goto err_mask; + + check_nonzero_interrupt_status(xdev); + /* explicitely zero all interrupt enable masks */ + channel_interrupts_disable(xdev, ~0); + user_interrupts_disable(xdev, ~0); + read_interrupts(xdev); + + rv = probe_engines(xdev); + if (rv) + goto err_engines; + + rv = enable_msi_msix(xdev, pdev); + if (rv < 0) + goto err_enable_msix; + + rv = irq_setup(xdev, pdev); + if (rv < 0) + goto err_interrupts; + + if (!poll_mode) + channel_interrupts_enable(xdev, ~0); + + /* Flush writes */ + read_interrupts(xdev); + + *user_max = xdev->user_max; + *h2c_channel_max = xdev->h2c_channel_max; + *c2h_channel_max = xdev->c2h_channel_max; + + xdma_device_flag_clear(xdev, XDEV_FLAG_OFFLINE); + return (void *)xdev; + +err_interrupts: + irq_teardown(xdev); +err_enable_msix: + disable_msi_msix(xdev, pdev); +err_engines: + remove_engines(xdev); +err_mask: + unmap_bars(xdev, pdev); +err_map: + if (xdev->got_regions) + pci_release_regions(pdev); +err_regions: + if (!xdev->regions_in_use) + pci_disable_device(pdev); +err_enable: + xdev_list_remove(xdev); + kfree(xdev); + return NULL; +} +EXPORT_SYMBOL_GPL(xdma_device_open); + +void xdma_device_close(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + dbg_init("pdev 0x%p, xdev 0x%p.\n", pdev, dev_hndl); + + if (!dev_hndl) + return; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return; + + dbg_sg("remove(dev = 0x%p) where pdev->dev.driver_data = 0x%p\n", + pdev, xdev); + if (xdev->pdev != pdev) { + dbg_sg("pci_dev(0x%lx) != pdev(0x%lx)\n", + (unsigned long)xdev->pdev, (unsigned long)pdev); + } + + channel_interrupts_disable(xdev, ~0); + user_interrupts_disable(xdev, ~0); + read_interrupts(xdev); + + irq_teardown(xdev); + disable_msi_msix(xdev, pdev); + + remove_engines(xdev); + unmap_bars(xdev, pdev); + + if (xdev->got_regions) { + dbg_init("pci_release_regions 0x%p.\n", pdev); + pci_release_regions(pdev); + } + + if (!xdev->regions_in_use) { + dbg_init("pci_disable_device 0x%p.\n", pdev); + pci_disable_device(pdev); + } + + xdev_list_remove(xdev); + + kfree(xdev); +} +EXPORT_SYMBOL_GPL(xdma_device_close); + +void xdma_device_offline(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + struct xdma_engine *engine; + int i; + + if (!dev_hndl) + return; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return; + +pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); + xdma_device_flag_set(xdev, XDEV_FLAG_OFFLINE); + + /* wait for all engines to be idle */ + for (i = 0; i < xdev->h2c_channel_max; i++) { + unsigned long flags; + + engine = &xdev->engine_h2c[i]; + + if (engine->magic == MAGIC_ENGINE) { + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown |= ENGINE_SHUTDOWN_REQUEST; + + xdma_engine_stop(engine); + engine->running = 0; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + for (i = 0; i < xdev->c2h_channel_max; i++) { + unsigned long flags; + + engine = &xdev->engine_c2h[i]; + if (engine->magic == MAGIC_ENGINE) { + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown |= ENGINE_SHUTDOWN_REQUEST; + + xdma_engine_stop(engine); + engine->running = 0; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + /* turn off interrupts */ + channel_interrupts_disable(xdev, ~0); + user_interrupts_disable(xdev, ~0); + read_interrupts(xdev); + irq_teardown(xdev); + + pr_info("xdev 0x%p, done.\n", xdev); +} +EXPORT_SYMBOL_GPL(xdma_device_offline); + +void xdma_device_online(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + struct xdma_engine *engine; + unsigned long flags; + int i; + + if (!dev_hndl) + return; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return; + +pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); + + for (i = 0; i < xdev->h2c_channel_max; i++) { + engine = &xdev->engine_h2c[i]; + if (engine->magic == MAGIC_ENGINE) { + engine_init_regs(engine); + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown &= ~ENGINE_SHUTDOWN_REQUEST; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + for (i = 0; i < xdev->c2h_channel_max; i++) { + engine = &xdev->engine_c2h[i]; + if (engine->magic == MAGIC_ENGINE) { + engine_init_regs(engine); + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown &= ~ENGINE_SHUTDOWN_REQUEST; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + /* re-write the interrupt table */ + if (!poll_mode) { + irq_setup(xdev, pdev); + + channel_interrupts_enable(xdev, ~0); + user_interrupts_enable(xdev, xdev->mask_irq_user); + read_interrupts(xdev); + } + + xdma_device_flag_clear(xdev, XDEV_FLAG_OFFLINE); +pr_info("xdev 0x%p, done.\n", xdev); +} +EXPORT_SYMBOL_GPL(xdma_device_online); + +int xdma_device_restart(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return -EINVAL; + + pr_info("NOT implemented, 0x%p.\n", xdev); + return -EINVAL; +} +EXPORT_SYMBOL_GPL(xdma_device_restart); + +int xdma_user_isr_register(void *dev_hndl, unsigned int mask, + irq_handler_t handler, void *dev) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + int i; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + for (i = 0; i < xdev->user_max && mask; i++) { + unsigned int bit = (1 << i); + + if ((bit & mask) == 0) + continue; + + mask &= ~bit; + xdev->user_irq[i].handler = handler; + xdev->user_irq[i].dev = dev; + } + + return 0; +} +EXPORT_SYMBOL_GPL(xdma_user_isr_register); + +int xdma_user_isr_enable(void *dev_hndl, unsigned int mask) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + xdev->mask_irq_user |= mask; + /* enable user interrupts */ + user_interrupts_enable(xdev, mask); + read_interrupts(xdev); + + return 0; +} +EXPORT_SYMBOL_GPL(xdma_user_isr_enable); + +int xdma_user_isr_disable(void *dev_hndl, unsigned int mask) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + xdev->mask_irq_user &= ~mask; + user_interrupts_disable(xdev, mask); + read_interrupts(xdev); + + return 0; +} +EXPORT_SYMBOL_GPL(xdma_user_isr_disable); + +#ifdef __LIBXDMA_MOD__ +static int __init xdma_base_init(void) +{ + printk(KERN_INFO "%s", version); + return 0; +} + +static void __exit xdma_base_exit(void) +{ + return; +} + +module_init(xdma_base_init); +module_exit(xdma_base_exit); +#endif +/* makes an existing transfer cyclic */ +static void xdma_transfer_cyclic(struct xdma_transfer *transfer) +{ + /* link last descriptor to first descriptor */ + xdma_desc_link(transfer->desc_virt + transfer->desc_num - 1, + transfer->desc_virt, transfer->desc_bus); + /* remember transfer is cyclic */ + transfer->cyclic = 1; +} + +static int transfer_monitor_cyclic(struct xdma_engine *engine, + struct xdma_transfer *transfer, int timeout_ms) +{ + struct xdma_result *result; + int rc = 0; + + BUG_ON(!engine); + BUG_ON(!transfer); + + result = engine->cyclic_result; + BUG_ON(!result); + + if (poll_mode) { + int i ; + for (i = 0; i < 5; i++) { + rc = engine_service_poll(engine, 0); + if (rc) { + pr_info("%s service_poll failed %d.\n", + engine->name, rc); + rc = -ERESTARTSYS; + } + if (result[engine->rx_head].status) + return 0; + } + } else { + if (enable_credit_mp){ + dbg_tfr("%s: rx_head=%d,rx_tail=%d, wait ...\n", + engine->name, engine->rx_head, engine->rx_tail); + rc = wait_event_interruptible_timeout( transfer->wq, + (engine->rx_head!=engine->rx_tail || + engine->rx_overrun), + msecs_to_jiffies(timeout_ms)); + dbg_tfr("%s: wait returns %d, rx %d/%d, overrun %d.\n", + engine->name, rc, engine->rx_head, + engine->rx_tail, engine->rx_overrun); + } else { + rc = wait_event_interruptible_timeout( transfer->wq, + engine->eop_found, + msecs_to_jiffies(timeout_ms)); + dbg_tfr("%s: wait returns %d, eop_found %d.\n", + engine->name, rc, engine->eop_found); + } + } + + return 0; +} + +struct scatterlist *sglist_index(struct sg_table *sgt, unsigned int idx) +{ + struct scatterlist *sg = sgt->sgl; + int i; + + if (idx >= sgt->orig_nents) + return NULL; + + if (!idx) + return sg; + + for (i = 0; i < idx; i++, sg = sg_next(sg)) + ; + + return sg; +} + +static int copy_cyclic_to_user(struct xdma_engine *engine, int pkt_length, + int head, char __user *buf, size_t count) +{ + struct scatterlist *sg; + int more = pkt_length; + + BUG_ON(!engine); + BUG_ON(!buf); + + dbg_tfr("%s, pkt_len %d, head %d, user buf idx %u.\n", + engine->name, pkt_length, head, engine->user_buffer_index); + + sg = sglist_index(&engine->cyclic_sgt, head); + if (!sg) { + pr_info("%s, head %d OOR, sgl %u.\n", + engine->name, head, engine->cyclic_sgt.orig_nents); + return -EIO; + } + + /* EOP found? Transfer anything from head to EOP */ + while (more) { + unsigned int copy = more > PAGE_SIZE ? PAGE_SIZE : more; + unsigned int blen = count - engine->user_buffer_index; + int rv; + + if (copy > blen) + copy = blen; + + dbg_tfr("%s sg %d, 0x%p, copy %u to user %u.\n", + engine->name, head, sg, copy, + engine->user_buffer_index); + + rv = copy_to_user(&buf[engine->user_buffer_index], + page_address(sg_page(sg)), copy); + if (rv) { + pr_info("%s copy_to_user %u failed %d\n", + engine->name, copy, rv); + return -EIO; + } + + more -= copy; + engine->user_buffer_index += copy; + + if (engine->user_buffer_index == count) { + /* user buffer used up */ + break; + } + + head++; + if (head >= CYCLIC_RX_PAGES_MAX) { + head = 0; + sg = engine->cyclic_sgt.sgl; + } else + sg = sg_next(sg); + } + + return pkt_length; +} + +static int complete_cyclic(struct xdma_engine *engine, char __user *buf, + size_t count) +{ + struct xdma_result *result; + int pkt_length = 0; + int fault = 0; + int eop = 0; + int head; + int rc = 0; + int num_credit = 0; + unsigned long flags; + + BUG_ON(!engine); + result = engine->cyclic_result; + BUG_ON(!result); + + spin_lock_irqsave(&engine->lock, flags); + + /* where the host currently is in the ring buffer */ + head = engine->rx_head; + + /* iterate over newly received results */ + while (engine->rx_head != engine->rx_tail||engine->rx_overrun) { + + WARN_ON(result[engine->rx_head].status==0); + + dbg_tfr("%s, result[%d].status = 0x%x length = 0x%x.\n", + engine->name, engine->rx_head, + result[engine->rx_head].status, + result[engine->rx_head].length); + + if ((result[engine->rx_head].status >> 16) != C2H_WB) { + pr_info("%s, result[%d].status 0x%x, no magic.\n", + engine->name, engine->rx_head, + result[engine->rx_head].status); + fault = 1; + } else if (result[engine->rx_head].length > PAGE_SIZE) { + pr_info("%s, result[%d].len 0x%x, > PAGE_SIZE 0x%lx.\n", + engine->name, engine->rx_head, + result[engine->rx_head].length, PAGE_SIZE); + fault = 1; + } else if (result[engine->rx_head].length == 0) { + pr_info("%s, result[%d].length 0x%x.\n", + engine->name, engine->rx_head, + result[engine->rx_head].length); + fault = 1; + /* valid result */ + } else { + pkt_length += result[engine->rx_head].length; + num_credit++; + /* seen eop? */ + //if (result[engine->rx_head].status & RX_STATUS_EOP) + if (result[engine->rx_head].status & RX_STATUS_EOP){ + eop = 1; + engine->eop_found = 1; + } + + dbg_tfr("%s, pkt_length=%d (%s)\n", + engine->name, pkt_length, + eop ? "with EOP" : "no EOP yet"); + } + /* clear result */ + result[engine->rx_head].status = 0; + result[engine->rx_head].length = 0; + /* proceed head pointer so we make progress, even when fault */ + engine->rx_head = (engine->rx_head + 1) % CYCLIC_RX_PAGES_MAX; + + /* stop processing if a fault/eop was detected */ + if (fault || eop){ + break; + } + } + + spin_unlock_irqrestore(&engine->lock, flags); + + if (fault) + return -EIO; + + rc = copy_cyclic_to_user(engine, pkt_length, head, buf, count); + engine->rx_overrun = 0; + /* if copy is successful, release credits */ + if(rc > 0) + write_register(num_credit,&engine->sgdma_regs->credits, 0); + + return rc; +} + +ssize_t xdma_engine_read_cyclic(struct xdma_engine *engine, char __user *buf, + size_t count, int timeout_ms) +{ + int i = 0; + int rc = 0; + int rc_len = 0; + struct xdma_transfer *transfer; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + transfer = &engine->cyclic_req->xfer; + BUG_ON(!transfer); + + engine->user_buffer_index = 0; + + do { + rc = transfer_monitor_cyclic(engine, transfer, timeout_ms); + if (rc < 0) + return rc; + rc = complete_cyclic(engine, buf, count); + if (rc < 0) + return rc; + rc_len += rc; + + i++; + if (i > 10) + break; + } while (!engine->eop_found); + + if(enable_credit_mp) + engine->eop_found = 0; + + return rc_len; +} + +static void sgt_free_with_pages(struct sg_table *sgt, int dir, + struct pci_dev *pdev) +{ + struct scatterlist *sg = sgt->sgl; + int npages = sgt->orig_nents; + int i; + + for (i = 0; i < npages; i++, sg = sg_next(sg)) { + struct page *pg = sg_page(sg); + dma_addr_t bus = sg_dma_address(sg); + + if (pg) { + if (pdev) + pci_unmap_page(pdev, bus, PAGE_SIZE, dir); + __free_page(pg); + } else + break; + } + sg_free_table(sgt); + memset(sgt, 0, sizeof(struct sg_table)); +} + +static int sgt_alloc_with_pages(struct sg_table *sgt, unsigned int npages, + int dir, struct pci_dev *pdev) +{ + struct scatterlist *sg; + int i; + + if (sg_alloc_table(sgt, npages, GFP_KERNEL)) { + pr_info("sgt OOM.\n"); + return -ENOMEM; + } + + sg = sgt->sgl; + for (i = 0; i < npages; i++, sg = sg_next(sg)) { + struct page *pg = alloc_page(GFP_KERNEL); + + if (!pg) { + pr_info("%d/%u, page OOM.\n", i, npages); + goto err_out; + } + + if (pdev) { + dma_addr_t bus = pci_map_page(pdev, pg, 0, PAGE_SIZE, + dir); + if (unlikely(pci_dma_mapping_error(pdev, bus))) { + pr_info("%d/%u, page 0x%p map err.\n", + i, npages, pg); + __free_page(pg); + goto err_out; + } + sg_dma_address(sg) = bus; + sg_dma_len(sg) = PAGE_SIZE; + } + sg_set_page(sg, pg, PAGE_SIZE, 0); + } + + sgt->orig_nents = sgt->nents = npages; + + return 0; + +err_out: + sgt_free_with_pages(sgt, dir, pdev); + return -ENOMEM; +} + +/* + * !NOTE! reference/demo purpose only + * xdma_cyclic_transfer_setup is used for streaming C2H transfers: + * - A list of buffers are pre-allocated for incoming streaming data + * - the ring of the buffers is allowed to wrap around + */ +int xdma_cyclic_transfer_setup(struct xdma_engine *engine) +{ + struct xdma_dev *xdev; + struct xdma_transfer *xfer; + dma_addr_t bus; + unsigned long flags; + int i; + int rc; + + BUG_ON(!engine); + xdev = engine->xdev; + BUG_ON(!xdev); + + if (engine->cyclic_req) { + pr_info("%s: exclusive access already taken.\n", + engine->name); + return -EBUSY; + } + + spin_lock_irqsave(&engine->lock, flags); + + engine->rx_tail = 0; + engine->rx_head = 0; + engine->rx_overrun = 0; + engine->eop_found = 0; + + rc = sgt_alloc_with_pages(&engine->cyclic_sgt, CYCLIC_RX_PAGES_MAX, + engine->dir, xdev->pdev); + if (rc < 0) { + pr_info("%s cyclic pages %u OOM.\n", + engine->name, CYCLIC_RX_PAGES_MAX); + goto err_out; + } + + engine->cyclic_req = xdma_init_request(&engine->cyclic_sgt, 0); + if (!engine->cyclic_req) { + pr_info("%s cyclic request OOM.\n", engine->name); + rc = -ENOMEM; + goto err_out; + } + +#ifdef __LIBXDMA_DEBUG__ + xdma_request_cb_dump(engine->cyclic_req); +#endif + + rc = transfer_init(engine, engine->cyclic_req); + if (rc < 0) + goto err_out; + + xfer = &engine->cyclic_req->xfer; + + /* replace source addresses with result write-back addresses */ + memset(engine->cyclic_result, 0, + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result)); + bus = engine->cyclic_result_bus; + for (i = 0; i < xfer->desc_num; i++) { + xfer->desc_virt[i].src_addr_lo = cpu_to_le32(PCI_DMA_L(bus)); + xfer->desc_virt[i].src_addr_hi = cpu_to_le32(PCI_DMA_H(bus)); + bus += sizeof(struct xdma_result); + } + /* set control of all descriptors */ + for (i = 0; i < xfer->desc_num; i++) { + xdma_desc_control_clear(xfer->desc_virt + i, LS_BYTE_MASK); + xdma_desc_control_set(xfer->desc_virt + i, + XDMA_DESC_EOP | XDMA_DESC_COMPLETED); + } + + /* make this a cyclic transfer */ + xdma_transfer_cyclic(xfer); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); +#endif + + if(enable_credit_mp){ + //write_register(RX_BUF_PAGES,&engine->sgdma_regs->credits); + write_register(128, &engine->sgdma_regs->credits, 0); + } + + spin_unlock_irqrestore(&engine->lock, flags); + + /* start cyclic transfer */ + transfer_queue(engine, xfer); + + return 0; + + /* unwind on errors */ +err_out: + if (engine->cyclic_req) { + xdma_request_free(engine->cyclic_req); + engine->cyclic_req = NULL; + } + + if (engine->cyclic_sgt.orig_nents) { + sgt_free_with_pages(&engine->cyclic_sgt, engine->dir, + xdev->pdev); + engine->cyclic_sgt.orig_nents = 0; + engine->cyclic_sgt.nents = 0; + engine->cyclic_sgt.sgl = NULL; + } + + spin_unlock_irqrestore(&engine->lock, flags); + + return rc; +} + + +static int cyclic_shutdown_polled(struct xdma_engine *engine) +{ + BUG_ON(!engine); + + spin_lock(&engine->lock); + + dbg_tfr("Polling for shutdown completion\n"); + do { + engine_status_read(engine, 1, 0); + schedule(); + } while (engine->status & XDMA_STAT_BUSY); + + if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { + dbg_tfr("Engine has stopped\n"); + + if (!list_empty(&engine->transfer_list)) + engine_transfer_dequeue(engine); + + engine_service_shutdown(engine); + } + + dbg_tfr("Shutdown completion polling done\n"); + spin_unlock(&engine->lock); + + return 0; +} + +static int cyclic_shutdown_interrupt(struct xdma_engine *engine) +{ + int rc; + + BUG_ON(!engine); + + rc = wait_event_interruptible_timeout(engine->shutdown_wq, + !engine->running, msecs_to_jiffies(10000)); + +#if 0 + if (rc) { + dbg_tfr("wait_event_interruptible=%d\n", rc); + return rc; + } +#endif + + if (engine->running) { + pr_info("%s still running?!, %d\n", engine->name, rc); + return -EINVAL; + } + + return rc; +} + +int xdma_cyclic_transfer_teardown(struct xdma_engine *engine) +{ + int rc; + struct xdma_dev *xdev = engine->xdev; + struct xdma_transfer *transfer; + unsigned long flags; + + transfer = engine_cyclic_stop(engine); + + spin_lock_irqsave(&engine->lock, flags); + if (transfer) { + dbg_tfr("%s: stop transfer 0x%p.\n", engine->name, transfer); + if (transfer != &engine->cyclic_req->xfer) { + pr_info("%s unexpected transfer 0x%p/0x%p\n", + engine->name, transfer, + &engine->cyclic_req->xfer); + } + } + /* allow engine to be serviced after stop request */ + spin_unlock_irqrestore(&engine->lock, flags); + + /* wait for engine to be no longer running */ + if (poll_mode) + rc = cyclic_shutdown_polled(engine); + else + rc = cyclic_shutdown_interrupt(engine); + + /* obtain spin lock to atomically remove resources */ + spin_lock_irqsave(&engine->lock, flags); + + if (engine->cyclic_req) { + xdma_request_free(engine->cyclic_req); + engine->cyclic_req = NULL; + } + + if (engine->cyclic_sgt.orig_nents) { + sgt_free_with_pages(&engine->cyclic_sgt, engine->dir, + xdev->pdev); + engine->cyclic_sgt.orig_nents = 0; + engine->cyclic_sgt.nents = 0; + engine->cyclic_sgt.sgl = NULL; + } + + spin_unlock_irqrestore(&engine->lock, flags); + + return 0; +} + +int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg) +{ + int rv; + unsigned long dst; + u32 w = XDMA_CTRL_NON_INCR_ADDR; + + dbg_perf("IOCTL_XDMA_ADDRMODE_SET\n"); + rv = get_user(dst, (int __user *)arg); + + if (rv == 0) { + engine->non_incr_addr = !!dst; + if (engine->non_incr_addr) + write_register(w, &engine->regs->control_w1s, + (unsigned long)(&engine->regs->control_w1s) - + (unsigned long)(&engine->regs)); + else + write_register(w, &engine->regs->control_w1c, + (unsigned long)(&engine->regs->control_w1c) - + (unsigned long)(&engine->regs)); + } + engine_alignments(engine); + + return rv; +} + diff --git a/sources/xdma_driver/libxdma/libxdma.h b/sources/xdma_driver/libxdma/libxdma.h new file mode 100644 index 0000000..bbf1a45 --- /dev/null +++ b/sources/xdma_driver/libxdma/libxdma.h @@ -0,0 +1,601 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef XDMA_LIB_H +#define XDMA_LIB_H + +#include <linux/version.h> +#include <linux/types.h> +#include <linux/uaccess.h> +#include <linux/module.h> +#include <linux/dma-mapping.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/jiffies.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/workqueue.h> + +/* Switch debug printing on/off */ +#define XDMA_DEBUG 0 + +/* SECTION: Preprocessor macros/constants */ +#define XDMA_BAR_NUM (6) + +/* maximum amount of register space to map */ +#define XDMA_BAR_SIZE (0x8000UL) + +/* Use this definition to poll several times between calls to schedule */ +#define NUM_POLLS_PER_SCHED 100 + +#define XDMA_CHANNEL_NUM_MAX (4) +/* + * interrupts per engine, rad2_vul.sv:237 + * .REG_IRQ_OUT (reg_irq_from_ch[(channel*2) +: 2]), + */ +#define XDMA_ENG_IRQ_NUM (1) +#define MAX_EXTRA_ADJ (15) +#define RX_STATUS_EOP (1) + +/* Target internal components on XDMA control BAR */ +#define XDMA_OFS_INT_CTRL (0x2000UL) +#define XDMA_OFS_CONFIG (0x3000UL) + +/* maximum number of desc per transfer request */ +#define XDMA_TRANSFER_MAX_DESC (2048) + +/* maximum size of a single DMA transfer descriptor */ +#define XDMA_DESC_BLEN_BITS 28 +#define XDMA_DESC_BLEN_MAX ((1 << (XDMA_DESC_BLEN_BITS)) - 1) + +/* bits of the SG DMA control register */ +#define XDMA_CTRL_RUN_STOP (1UL << 0) +#define XDMA_CTRL_IE_DESC_STOPPED (1UL << 1) +#define XDMA_CTRL_IE_DESC_COMPLETED (1UL << 2) +#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH (1UL << 3) +#define XDMA_CTRL_IE_MAGIC_STOPPED (1UL << 4) +#define XDMA_CTRL_IE_IDLE_STOPPED (1UL << 6) +#define XDMA_CTRL_IE_READ_ERROR (0x1FUL << 9) +#define XDMA_CTRL_IE_DESC_ERROR (0x1FUL << 19) +#define XDMA_CTRL_NON_INCR_ADDR (1UL << 25) +#define XDMA_CTRL_POLL_MODE_WB (1UL << 26) + +/* bits of the SG DMA status register */ +#define XDMA_STAT_BUSY (1UL << 0) +#define XDMA_STAT_DESC_STOPPED (1UL << 1) +#define XDMA_STAT_DESC_COMPLETED (1UL << 2) +#define XDMA_STAT_ALIGN_MISMATCH (1UL << 3) +#define XDMA_STAT_MAGIC_STOPPED (1UL << 4) +#define XDMA_STAT_INVALID_LEN (1UL << 5) +#define XDMA_STAT_IDLE_STOPPED (1UL << 6) + +#define XDMA_STAT_COMMON_ERR_MASK \ + (XDMA_STAT_ALIGN_MISMATCH | XDMA_STAT_MAGIC_STOPPED | \ + XDMA_STAT_INVALID_LEN) + +/* desc_error, C2H & H2C */ +#define XDMA_STAT_DESC_UNSUPP_REQ (1UL << 19) +#define XDMA_STAT_DESC_COMPL_ABORT (1UL << 20) +#define XDMA_STAT_DESC_PARITY_ERR (1UL << 21) +#define XDMA_STAT_DESC_HEADER_EP (1UL << 22) +#define XDMA_STAT_DESC_UNEXP_COMPL (1UL << 23) + +#define XDMA_STAT_DESC_ERR_MASK \ + (XDMA_STAT_DESC_UNSUPP_REQ | XDMA_STAT_DESC_COMPL_ABORT | \ + XDMA_STAT_DESC_PARITY_ERR | XDMA_STAT_DESC_HEADER_EP | \ + XDMA_STAT_DESC_UNEXP_COMPL) + +/* read error: H2C */ +#define XDMA_STAT_H2C_R_UNSUPP_REQ (1UL << 9) +#define XDMA_STAT_H2C_R_COMPL_ABORT (1UL << 10) +#define XDMA_STAT_H2C_R_PARITY_ERR (1UL << 11) +#define XDMA_STAT_H2C_R_HEADER_EP (1UL << 12) +#define XDMA_STAT_H2C_R_UNEXP_COMPL (1UL << 13) + +#define XDMA_STAT_H2C_R_ERR_MASK \ + (XDMA_STAT_H2C_R_UNSUPP_REQ | XDMA_STAT_H2C_R_COMPL_ABORT | \ + XDMA_STAT_H2C_R_PARITY_ERR | XDMA_STAT_H2C_R_HEADER_EP | \ + XDMA_STAT_H2C_R_UNEXP_COMPL) + +/* write error, H2C only */ +#define XDMA_STAT_H2C_W_DECODE_ERR (1UL << 14) +#define XDMA_STAT_H2C_W_SLAVE_ERR (1UL << 15) + +#define XDMA_STAT_H2C_W_ERR_MASK \ + (XDMA_STAT_H2C_W_DECODE_ERR | XDMA_STAT_H2C_W_SLAVE_ERR) + +/* read error: C2H */ +#define XDMA_STAT_C2H_R_DECODE_ERR (1UL << 9) +#define XDMA_STAT_C2H_R_SLAVE_ERR (1UL << 10) + +#define XDMA_STAT_C2H_R_ERR_MASK \ + (XDMA_STAT_C2H_R_DECODE_ERR | XDMA_STAT_C2H_R_SLAVE_ERR) + +/* all combined */ +#define XDMA_STAT_H2C_ERR_MASK \ + (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \ + XDMA_STAT_H2C_R_ERR_MASK | XDMA_STAT_H2C_W_ERR_MASK) + +#define XDMA_STAT_C2H_ERR_MASK \ + (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \ + XDMA_STAT_C2H_R_ERR_MASK) + +/* bits of the SGDMA descriptor control field */ +#define XDMA_DESC_STOPPED (1UL << 0) +#define XDMA_DESC_COMPLETED (1UL << 1) +#define XDMA_DESC_EOP (1UL << 4) + +#define XDMA_PERF_RUN (1UL << 0) +#define XDMA_PERF_CLEAR (1UL << 1) +#define XDMA_PERF_AUTO (1UL << 2) + +#define MAGIC_ENGINE 0xEEEEEEEEUL +#define MAGIC_DEVICE 0xDDDDDDDDUL + +/* upper 16-bits of engine identifier register */ +#define XDMA_ID_H2C 0x1fc0U +#define XDMA_ID_C2H 0x1fc1U + +/* for C2H AXI-ST mode */ +#define CYCLIC_RX_PAGES_MAX 256 + +#define LS_BYTE_MASK 0x000000FFUL + +#define BLOCK_ID_MASK 0xFFF00000 +#define BLOCK_ID_HEAD 0x1FC00000 + +#define IRQ_BLOCK_ID 0x1fc20000UL +#define CONFIG_BLOCK_ID 0x1fc30000UL + +#define WB_COUNT_MASK 0x00ffffffUL +#define WB_ERR_MASK (1UL << 31) +#define POLL_TIMEOUT_SECONDS 10 + +#define MAX_USER_IRQ 16 + +#define MAX_DESC_BUS_ADDR (0xffffffffULL) + +#define DESC_MAGIC 0xAD4B0000UL + +#define C2H_WB 0x52B4UL + +#define MAX_NUM_ENGINES (XDMA_CHANNEL_NUM_MAX * 2) +#define H2C_CHANNEL_OFFSET 0x1000 +#define SGDMA_OFFSET_FROM_CHANNEL 0x4000 +#define CHANNEL_SPACING 0x100 +#define TARGET_SPACING 0x1000 + +#define BYPASS_MODE_SPACING 0x0100 + +/* obtain the 32 most significant (high) bits of a 32-bit or 64-bit address */ +#define PCI_DMA_H(addr) ((addr >> 16) >> 16) +/* obtain the 32 least significant (low) bits of a 32-bit or 64-bit address */ +#define PCI_DMA_L(addr) (addr & 0xffffffffUL) + +#ifndef VM_RESERVED + #define VMEM_FLAGS (VM_IO | VM_DONTEXPAND | VM_DONTDUMP) +#else + #define VMEM_FLAGS (VM_IO | VM_RESERVED) +#endif + +#ifdef __LIBXDMA_DEBUG__ +#define dbg_io pr_err +#define dbg_fops pr_err +#define dbg_perf pr_err +#define dbg_sg pr_err +#define dbg_tfr pr_err +#define dbg_irq pr_err +#define dbg_init pr_err +#define dbg_desc pr_err +#else +/* disable debugging */ +#define dbg_io(...) +#define dbg_fops(...) +#define dbg_perf(...) +#define dbg_sg(...) +#define dbg_tfr(...) +#define dbg_irq(...) +#define dbg_init(...) +#define dbg_desc(...) +#endif + +/* SECTION: Enum definitions */ +enum transfer_state { + TRANSFER_STATE_NEW = 0, + TRANSFER_STATE_SUBMITTED, + TRANSFER_STATE_COMPLETED, + TRANSFER_STATE_FAILED, + TRANSFER_STATE_ABORTED +}; + +enum shutdown_state { + ENGINE_SHUTDOWN_NONE = 0, /* No shutdown in progress */ + ENGINE_SHUTDOWN_REQUEST = 1, /* engine requested to shutdown */ + ENGINE_SHUTDOWN_IDLE = 2 /* engine has shutdown and is idle */ +}; + +enum dev_capabilities { + CAP_64BIT_DMA = 2, + CAP_64BIT_DESC = 4, + CAP_ENGINE_WRITE = 8, + CAP_ENGINE_READ = 16 +}; + +/* SECTION: Structure definitions */ + +struct config_regs { + u32 identifier; + u32 reserved_1[4]; + u32 msi_enable; +}; + +/** + * SG DMA Controller status and control registers + * + * These registers make the control interface for DMA transfers. + * + * It sits in End Point (FPGA) memory BAR[0] for 32-bit or BAR[0:1] for 64-bit. + * It references the first descriptor which exists in Root Complex (PC) memory. + * + * @note The registers must be accessed using 32-bit (PCI DWORD) read/writes, + * and their values are in little-endian byte ordering. + */ +struct engine_regs { + u32 identifier; + u32 control; + u32 control_w1s; + u32 control_w1c; + u32 reserved_1[12]; /* padding */ + + u32 status; + u32 status_rc; + u32 completed_desc_count; + u32 alignments; + u32 reserved_2[14]; /* padding */ + + u32 poll_mode_wb_lo; + u32 poll_mode_wb_hi; + u32 interrupt_enable_mask; + u32 interrupt_enable_mask_w1s; + u32 interrupt_enable_mask_w1c; + u32 reserved_3[9]; /* padding */ + + u32 perf_ctrl; + u32 perf_cyc_lo; + u32 perf_cyc_hi; + u32 perf_dat_lo; + u32 perf_dat_hi; + u32 perf_pnd_lo; + u32 perf_pnd_hi; +} __packed; + +struct engine_sgdma_regs { + u32 identifier; + u32 reserved_1[31]; /* padding */ + + /* bus address to first descriptor in Root Complex Memory */ + u32 first_desc_lo; + u32 first_desc_hi; + /* number of adjacent descriptors at first_desc */ + u32 first_desc_adjacent; + u32 credits; +} __packed; + +struct msix_vec_table_entry { + u32 msi_vec_addr_lo; + u32 msi_vec_addr_hi; + u32 msi_vec_data_lo; + u32 msi_vec_data_hi; +} __packed; + +struct msix_vec_table { + struct msix_vec_table_entry entry_list[32]; +} __packed; + +struct interrupt_regs { + u32 identifier; + u32 user_int_enable; + u32 user_int_enable_w1s; + u32 user_int_enable_w1c; + u32 channel_int_enable; + u32 channel_int_enable_w1s; + u32 channel_int_enable_w1c; + u32 reserved_1[9]; /* padding */ + + u32 user_int_request; + u32 channel_int_request; + u32 user_int_pending; + u32 channel_int_pending; + u32 reserved_2[12]; /* padding */ + + u32 user_msi_vector[8]; + u32 channel_msi_vector[8]; +} __packed; + +struct sgdma_common_regs { + u32 padding[8]; + u32 credit_mode_enable; + u32 credit_mode_enable_w1s; + u32 credit_mode_enable_w1c; +} __packed; + + +/* Structure for polled mode descriptor writeback */ +struct xdma_poll_wb { + u32 completed_desc_count; + u32 reserved_1[7]; +} __packed; + + +/** + * Descriptor for a single contiguous memory block transfer. + * + * Multiple descriptors are linked by means of the next pointer. An additional + * extra adjacent number gives the amount of extra contiguous descriptors. + * + * The descriptors are in root complex memory, and the bytes in the 32-bit + * words must be in little-endian byte ordering. + */ +struct xdma_desc { + u32 control; + u32 bytes; /* transfer length in bytes */ + u32 src_addr_lo; /* source address (low 32-bit) */ + u32 src_addr_hi; /* source address (high 32-bit) */ + u32 dst_addr_lo; /* destination address (low 32-bit) */ + u32 dst_addr_hi; /* destination address (high 32-bit) */ + /* + * next descriptor in the single-linked list of descriptors; + * this is the PCIe (bus) address of the next descriptor in the + * root complex memory + */ + u32 next_lo; /* next desc address (low 32-bit) */ + u32 next_hi; /* next desc address (high 32-bit) */ +} __packed; + +/* 32 bytes (four 32-bit words) or 64 bytes (eight 32-bit words) */ +struct xdma_result { + u32 status; + u32 length; + u32 reserved_1[6]; /* padding */ +} __packed; + +struct sw_desc { + dma_addr_t addr; + unsigned int len; +}; + +/* Describes a (SG DMA) single transfer for the engine */ +struct xdma_transfer { + struct list_head entry; /* queue of non-completed transfers */ + struct xdma_desc *desc_virt; /* virt addr of the 1st descriptor */ + dma_addr_t desc_bus; /* bus addr of the first descriptor */ + int desc_adjacent; /* adjacent descriptors at desc_bus */ + int desc_num; /* number of descriptors in transfer */ + enum dma_data_direction dir; + wait_queue_head_t wq; /* wait queue for transfer completion */ + + enum transfer_state state; /* state of the transfer */ + unsigned int flags; +#define XFER_FLAG_NEED_UNMAP 0x1 + int cyclic; /* flag if transfer is cyclic */ + int last_in_request; /* flag if last within request */ + unsigned int len; + struct sg_table *sgt; +}; + +struct xdma_request_cb { + struct sg_table *sgt; + unsigned int total_len; + u64 ep_addr; + + struct xdma_transfer xfer; + + unsigned int sw_desc_idx; + unsigned int sw_desc_cnt; + struct sw_desc sdesc[0]; +}; + +struct xdma_engine { + unsigned long magic; /* structure ID for sanity checks */ + struct xdma_dev *xdev; /* parent device */ + char name[5]; /* name of this engine */ + int version; /* version of this engine */ + //dev_t cdevno; /* character device major:minor */ + //struct cdev cdev; /* character device (embedded struct) */ + + /* HW register address offsets */ + struct engine_regs *regs; /* Control reg BAR offset */ + struct engine_sgdma_regs *sgdma_regs; /* SGDAM reg BAR offset */ + u32 bypass_offset; /* Bypass mode BAR offset */ + + /* Engine state, configuration and flags */ + enum shutdown_state shutdown; /* engine shutdown mode */ + enum dma_data_direction dir; + int device_open; /* flag if engine node open, ST mode only */ + int running; /* flag if the driver started engine */ + int non_incr_addr; /* flag if non-incremental addressing used */ + int streaming; + int addr_align; /* source/dest alignment in bytes */ + int len_granularity; /* transfer length multiple */ + int addr_bits; /* HW datapath address width */ + int channel; /* engine indices */ + int max_extra_adj; /* descriptor prefetch capability */ + int desc_dequeued; /* num descriptors of completed transfers */ + u32 status; /* last known status of device */ + u32 interrupt_enable_mask_value;/* only used for MSIX mode to store per-engine interrupt mask value */ + + /* Transfer list management */ + struct list_head transfer_list; /* queue of transfers */ + + /* Members applicable to AXI-ST C2H (cyclic) transfers */ + struct xdma_result *cyclic_result; + dma_addr_t cyclic_result_bus; /* bus addr for transfer */ + struct xdma_request_cb *cyclic_req; + struct sg_table cyclic_sgt; + u8 eop_found; /* used only for cyclic(rx:c2h) */ + + int rx_tail; /* follows the HW */ + int rx_head; /* where the SW reads from */ + int rx_overrun; /* flag if overrun occured */ + + /* for copy from cyclic buffer to user buffer */ + unsigned int user_buffer_index; + + /* Members associated with polled mode support */ + u8 *poll_mode_addr_virt; /* virt addr for descriptor writeback */ + dma_addr_t poll_mode_bus; /* bus addr for descriptor writeback */ + + /* Members associated with interrupt mode support */ + wait_queue_head_t shutdown_wq; /* wait queue for shutdown sync */ + spinlock_t lock; /* protects concurrent access */ + int prev_cpu; /* remember CPU# of (last) locker */ + int msix_irq_line; /* MSI-X vector for this engine */ + u32 irq_bitmask; /* IRQ bit mask for this engine */ + struct work_struct work; /* Work queue for interrupt handling */ + + spinlock_t desc_lock; /* protects concurrent access */ + dma_addr_t desc_bus; + struct xdma_desc *desc; + + /* for performance test support */ + struct xdma_performance_ioctl *xdma_perf; /* perf test control */ + wait_queue_head_t xdma_perf_wq; /* Perf test sync */ +}; + +struct xdma_user_irq { + struct xdma_dev *xdev; /* parent device */ + u8 user_idx; /* 0 ~ 15 */ + u8 events_irq; /* accumulated IRQs */ + spinlock_t events_lock; /* lock to safely update events_irq */ + wait_queue_head_t events_wq; /* wait queue to sync waiting threads */ + irq_handler_t handler; + + void *dev; +}; + +/* XDMA PCIe device specific book-keeping */ +#define XDEV_FLAG_OFFLINE 0x1 +struct xdma_dev { + struct list_head list_head; + struct list_head rcu_node; + + unsigned long magic; /* structure ID for sanity checks */ + struct pci_dev *pdev; /* pci device struct from probe() */ + int idx; /* dev index */ + + const char *mod_name; /* name of module owning the dev */ + + spinlock_t lock; /* protects concurrent access */ + unsigned int flags; + + /* PCIe BAR management */ + void *__iomem bar[XDMA_BAR_NUM]; /* addresses for mapped BARs */ + int user_bar_idx; /* BAR index of user logic */ + int config_bar_idx; /* BAR index of XDMA config logic */ + int bypass_bar_idx; /* BAR index of XDMA bypass logic */ + int regions_in_use; /* flag if dev was in use during probe() */ + int got_regions; /* flag if probe() obtained the regions */ + + int user_max; + int c2h_channel_max; + int h2c_channel_max; + + /* Interrupt management */ + int irq_count; /* interrupt counter */ + int irq_line; /* flag if irq allocated successfully */ + int msi_enabled; /* flag if msi was enabled for the device */ + int msix_enabled; /* flag if msi-x was enabled for the device */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,12,0) + struct msix_entry entry[32]; /* msi-x vector/entry table */ +#endif + struct xdma_user_irq user_irq[16]; /* user IRQ management */ + unsigned int mask_irq_user; + + /* XDMA engine management */ + int engines_num; /* Total engine count */ + u32 mask_irq_h2c; + u32 mask_irq_c2h; + struct xdma_engine engine_h2c[XDMA_CHANNEL_NUM_MAX]; + struct xdma_engine engine_c2h[XDMA_CHANNEL_NUM_MAX]; + + /* SD_Accel specific */ + enum dev_capabilities capabilities; + u64 feature_id; +}; + +static inline int xdma_device_flag_check(struct xdma_dev *xdev, unsigned int f) +{ + unsigned long flags; + + spin_lock_irqsave(&xdev->lock, flags); + if (xdev->flags & f) { + spin_unlock_irqrestore(&xdev->lock, flags); + return 1; + } + spin_unlock_irqrestore(&xdev->lock, flags); + return 0; +} + +static inline int xdma_device_flag_test_n_set(struct xdma_dev *xdev, + unsigned int f) +{ + unsigned long flags; + int rv = 0; + + spin_lock_irqsave(&xdev->lock, flags); + if (xdev->flags & f) { + spin_unlock_irqrestore(&xdev->lock, flags); + rv = 1; + } else + xdev->flags |= f; + spin_unlock_irqrestore(&xdev->lock, flags); + return rv; +} + +static inline void xdma_device_flag_set(struct xdma_dev *xdev, unsigned int f) +{ + unsigned long flags; + + spin_lock_irqsave(&xdev->lock, flags); + xdev->flags |= f; + spin_unlock_irqrestore(&xdev->lock, flags); +} + +static inline void xdma_device_flag_clear(struct xdma_dev *xdev, unsigned int f) +{ + unsigned long flags; + + spin_lock_irqsave(&xdev->lock, flags); + xdev->flags &= ~f; + spin_unlock_irqrestore(&xdev->lock, flags); +} + +void write_register(u32 value, void *iomem); +u32 read_register(void *iomem); + +struct xdma_dev *xdev_find_by_pdev(struct pci_dev *pdev); + +void xdma_device_offline(struct pci_dev *pdev, void *dev_handle); +void xdma_device_online(struct pci_dev *pdev, void *dev_handle); + +int xdma_performance_submit(struct xdma_dev *xdev, struct xdma_engine *engine); +struct xdma_transfer *engine_cyclic_stop(struct xdma_engine *engine); +void enable_perf(struct xdma_engine *engine); +void get_perf_stats(struct xdma_engine *engine); + +int xdma_cyclic_transfer_setup(struct xdma_engine *engine); +int xdma_cyclic_transfer_teardown(struct xdma_engine *engine); +ssize_t xdma_engine_read_cyclic(struct xdma_engine *, char __user *, size_t, + int); +int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg); + +#endif /* XDMA_LIB_H */ diff --git a/sources/xdma_driver/libxdma/version.h b/sources/xdma_driver/libxdma/version.h new file mode 100644 index 0000000..703424d --- /dev/null +++ b/sources/xdma_driver/libxdma/version.h @@ -0,0 +1,28 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef __XDMA_VERSION_H__ +#define __XDMA_VERSION_H__ + +#define DRV_MOD_MAJOR 2017 +#define DRV_MOD_MINOR 1 +#define DRV_MOD_PATCHLEVEL 36 + +#define DRV_MODULE_VERSION \ + __stringify(DRV_MOD_MAJOR) "." \ + __stringify(DRV_MOD_MINOR) "." \ + __stringify(DRV_MOD_PATCHLEVEL) + +#define DRV_MOD_VERSION_NUMBER \ + ((DRV_MOD_MAJOR)*1000 + (DRV_MOD_MINOR)*100 + DRV_MOD_PATCHLEVEL) + +#endif /* ifndef __XDMA_VERSION_H__ */ diff --git a/sources/xdma_driver/load_driver.sh b/sources/xdma_driver/load_driver.sh new file mode 100755 index 0000000..9910136 --- /dev/null +++ b/sources/xdma_driver/load_driver.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Make sure only root can run our script +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" 1>&2 + exit 1 +fi + +cd xdma +make clean +make -j +if [ ! $? == 0 ]; then + echo "Error: Kernel module did not compile properly." + echo " FAILED" + exit 1 +fi +cd .. + + +# Remove the existing xdma kernel module +lsmod | grep xdma +if [ $? -eq 0 ]; then + rmmod xdma +fi +echo -n "Loading xdma driver..." +# Use the following command to Load the driver in the default +# or interrupt drive mode. This will allow the driver to use +# interrupts to signal when DMA transfers are completed. +insmod xdma/xdma.ko enable_credit_mp=1 +# Use the following command to Load the driver in Polling +# mode rather than than interrupt mode. This will allow the +# driver to use polling to determ when DMA transfers are +# completed. +#insmod ../xdma/xdma.ko poll_mode=1 + +if [ ! $? == 0 ]; then + echo "Error: Kernel module did not load properly." + echo " FAILED" + exit 1 +fi + +# Check to see if the xdma devices were recognized +echo "" +cat /proc/devices | grep xdma > /dev/null +returnVal=$? +if [ $returnVal == 0 ]; then + # Installed devices were recognized. + echo "The Kernel module installed correctly and the xmda devices were recognized." +else + # No devices were installed. + echo "Error: The Kernel module installed correctly, but no devices were recognized." + echo " FAILED" + exit 1 +fi + +echo " DONE" diff --git a/sources/xdma_driver/readme.txt b/sources/xdma_driver/readme.txt new file mode 100644 index 0000000..cbc80f0 --- /dev/null +++ b/sources/xdma_driver/readme.txt @@ -0,0 +1,140 @@ +Modifications to make the dirver compatible with newer Linux kernels: +===================================================================== +For kernel version >= 5.3.0: +- xdma/: + - cdev_ctrl.c: Interface changed for access_ok(): https://forums.xilinx.com/t5/PCIe-and-CPM/PCIe-DMA-driver-compilation-issues-in-Linux-Ubuntu-19-04/td-p/1022239 + + - cdev_xvc.c: + - libxdma.c: Does not need mmiowb() + +(This readme is less relevant as we've modified the file structure) + +The files in this directory provide Xilinx PCIe DMA drivers, example software, +and example test scripts that can be used to exercise the Xilinx PCIe DMA IP. + +This software can be used directly or referenced to create drivers and software +for your Xilinx FPGA hardware design. + +Directory and file description: +=============================== + - xdma/: This directory contains the Xilinx PCIe DMA kernel module + driver files. + + - libxdma/: This directory contains support files for the kernel driver module, + which interfaces directly with the XDMA IP. + + - include/: This directory contains all include files that are needed for + compiling driver. + + - etc/: This directory contains rules for the Xilinx PCIe DMA kernel module + and software. The files in this directory should be copied to the /etc/ + directory on your linux system. + + - tests/: This directory contains example application software to exercise the + provided kernel module driver and Xilinx PCIe DMA IP. This directory + also contains the following scripts and directories. + + - load_driver.sh: + This script loads the kernel module and creates the necissary + kernel nodes used by the provided software. + The The kernel device nodes will be created under /dev/xdma*. + Additional device nodes are created under /dev/xdma/card* to + more easily differentiate between multiple PCIe DMA enabled + cards. Root permissions will be required to run this script. + + - run_test.sh: + This script runs sample tests on a Xilinx PCIe DMA target and + returns a pass (0) or fail (1) result. + This script is intended for use with the PCIe DMA example + design. + + - perform_hwcount.sh: + This script runs hardware performance for XDMA for both Host to + Card (H2C) and Card to Host (C2H). The result are copied to + 'hw_log_h2c.txt' and hw_log_c2h.txt' text files. + For each direction the performance script loops from 64 bytes + to 4MBytes and generate performance numbers (byte size doubles + for each loop count). + You can grep for 'data rate' on those two files to see data + rate values. + Data rate values are in percentage of maximum throughput. + Maximum data rate for x8 Gen3 is 8Gbytes/s, so for a x8Gen3 + design value of 0.81 data rate is 0.81*8 = 6.48Gbytes/s. + Maximum data rate for x16 Gen3 is 16Gbytes/s, so for a x16Gen3 + design value of 0.78 data rate is 0.78*16 = 12.48Gbytes/s. + This program can be run on AXI-MM example design. + AXI-ST example design is a loopback design, both H2C and C2H + are connected. Running on AXI-ST example design will not + generate proper numbers. + If a AXI-ST design is independent of H2C and C2H, performance + number can be generated. + - data/: + This directory contains binary data files that are used for DMA + data transfers to the Xilinx FPGA PCIe endpoint device. + +Usage: + - Change directory to the driver directory. + cd xdma + - Compile and install the kernel module driver. + make install + - Change directory to the tools directory. + cd tools + - Compile the provided example test tools. + make + - Copy the provided driver rules from the etc directory to the /etc/ directory + on your system. + cp ../etc/udev/rules.d/* /etc/udev/rules.d/ + - Load the kernel module driver: + a. modprobe xdma + b. using the provided script. + cd tests + ./load_driver.sh + - Run the provided test script to generate basic DMA traffic. + ./run_test.sh + - Check driver Version number + modinfo xdma (or) + modinfo ../xdma/xdma.ko + +Updates and Backward Compaitiblity: + - The following features were added to the PCIe DMA IP and driver in Vivado + 2016.1. These features cannot be used with PCIe DMA IP if the IP was + generated using a Vivado build earlier than 2016.1. + - Poll Mode: Earlier versions of Vivado only support interrupt mode which + is the default behavior of the driver. + - Source/Destination Address: Earlier versions of Vivado PCIe DMA IP + required the low-order bits of the Source and Destination address to be + the same. + As of 2016.1 this restriction has been removed and the Source and + Destination addresses can be any arbitrary address that is valid for + your system. + +Frequently asked questions: + Q: How do I uninstall the kernel module driver? + A: Use the following commands to uninstall the driver. + - Uninstall the kernel module. + rmmod -s xdma + - Delete the dma rules that were added. + rm -f /etc/udev/rules.d/60-xdma.rules + rm -f /etc/udev/rules.d/xdma-udev-command.sh + + Q: How do I modify the PCIe Device IDs recognized by the kernel module driver? + A: The xdma/xdma_mod.c file constains the pci_device_id struct that identifies + the PCIe Device IDs that are recognized by the driver in the following + format: + { PCI_DEVICE(0x10ee, 0x8038), }, + Add, remove, or modify the PCIe Device IDs in this struct as desired. Then + uninstall the existing xdma kernel module, compile the driver again, and + re-install the driver using the load_driver.sh script. + + Q: By default the driver uses interupts to signal when DMA transfers are + completed. How do I modify the driver to use polling rather than + interrupts to determine when DMA transactions are completed? + A: The driver can be changed from being interrupt driven (default) to being + polling driven (poll mode) when the kernel module is inserted. To do this + modify the load_driver.sh file as follows: + Change: insmod xdma/xdma.ko + To: insmod xdma/xdma.ko poll_mode=1 + Note: Interrupt vs Poll mode will apply to all DMA channels. If desired the + driver can be modified such that some channels are interrupt driven while + others are polling driven. Refer to the poll mode section of PG195 for + additional information on using the PCIe DMA IP in poll mode. diff --git a/sources/xdma_driver/xdma/Makefile b/sources/xdma_driver/xdma/Makefile new file mode 100644 index 0000000..d1c07a8 --- /dev/null +++ b/sources/xdma_driver/xdma/Makefile @@ -0,0 +1,35 @@ +SHELL = /bin/bash +ifneq ($(xvc_bar_num),) + XVC_FLAGS += -D__XVC_BAR_NUM__=$(xvc_bar_num) +endif + +ifneq ($(xvc_bar_offset),) + XVC_FLAGS += -D__XVC_BAR_OFFSET__=$(xvc_bar_offset) +endif + +$(warning XVC_FLAGS: $(XVC_FLAGS).) + +topdir := $(shell cd $(src)/.. && pwd) + +TARGET_MODULE:=xdma + +EXTRA_CFLAGS := -I$(topdir)/include $(XVC_FLAGS) +#EXTRA_CFLAGS += -D__LIBXDMA_DEBUG__ +#EXTRA_CFLAGS += -DINTERNAL_TESTING + +ifneq ($(KERNELRELEASE),) + $(TARGET_MODULE)-objs := libxdma.o xdma_cdev.o cdev_ctrl.o cdev_events.o cdev_sgdma.o cdev_xvc.o cdev_bypass.o xdma_mod.o + obj-m := $(TARGET_MODULE).o +else + BUILDSYSTEM_DIR:=/lib/modules/$(shell uname -r)/build + PWD:=$(shell pwd) +all : + $(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) modules + +clean: + $(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) clean + +install: all + $(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) modules_install + +endif diff --git a/sources/xdma_driver/xdma/cdev_bypass.c b/sources/xdma_driver/xdma/cdev_bypass.c new file mode 100644 index 0000000..4b84526 --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_bypass.c @@ -0,0 +1,180 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "libxdma_api.h" +#include "xdma_cdev.h" + +#define write_register(v,mem,off) iowrite32(v, mem) + +static int copy_desc_data(struct xdma_transfer *transfer, char __user *buf, + size_t *buf_offset, size_t buf_size) +{ + int i; + int copy_err; + int rc = 0; + + BUG_ON(!buf); + BUG_ON(!buf_offset); + + /* Fill user buffer with descriptor data */ + for (i = 0; i < transfer->desc_num; i++) { + if (*buf_offset + sizeof(struct xdma_desc) <= buf_size) { + copy_err = copy_to_user(&buf[*buf_offset], + transfer->desc_virt + i, + sizeof(struct xdma_desc)); + + if (copy_err) { + dbg_sg("Copy to user buffer failed\n"); + *buf_offset = buf_size; + rc = -EINVAL; + } else { + *buf_offset += sizeof(struct xdma_desc); + } + } else { + rc = -ENOMEM; + } + } + + return rc; +} + +static ssize_t char_bypass_read(struct file *file, char __user *buf, + size_t count, loff_t *pos) +{ + struct xdma_dev *xdev; + struct xdma_engine *engine; + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + struct xdma_transfer *transfer; + struct list_head *idx; + size_t buf_offset = 0; + int rc = 0; + + rc = xcdev_check(__func__, xcdev, 1); + if (rc < 0) + return rc; + xdev = xcdev->xdev; + engine = xcdev->engine; + + dbg_sg("In char_bypass_read()\n"); + + if (count & 3) { + dbg_sg("Buffer size must be a multiple of 4 bytes\n"); + return -EINVAL; + } + + if (!buf) { + dbg_sg("Caught NULL pointer\n"); + return -EINVAL; + } + + if (xdev->bypass_bar_idx < 0) { + dbg_sg("Bypass BAR not present - unsupported operation\n"); + return -ENODEV; + } + + spin_lock(&engine->lock); + + if (!list_empty(&engine->transfer_list)) { + list_for_each(idx, &engine->transfer_list) { + transfer = list_entry(idx, struct xdma_transfer, entry); + + rc = copy_desc_data(transfer, buf, &buf_offset, count); + } + } + + spin_unlock(&engine->lock); + + if (rc < 0) + return rc; + else + return buf_offset; +} + +static ssize_t char_bypass_write(struct file *file, const char __user *buf, + size_t count, loff_t *pos) +{ + struct xdma_dev *xdev; + struct xdma_engine *engine; + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + + u32 desc_data; + u32 *bypass_addr; + size_t buf_offset = 0; + int rc = 0; + int copy_err; + + rc = xcdev_check(__func__, xcdev, 1); + if (rc < 0) + return rc; + xdev = xcdev->xdev; + engine = xcdev->engine; + + if (count & 3) { + dbg_sg("Buffer size must be a multiple of 4 bytes\n"); + return -EINVAL; + } + + if (!buf) { + dbg_sg("Caught NULL pointer\n"); + return -EINVAL; + } + + if (xdev->bypass_bar_idx < 0) { + dbg_sg("Bypass BAR not present - unsupported operation\n"); + return -ENODEV; + } + + dbg_sg("In char_bypass_write()\n"); + + spin_lock(&engine->lock); + + /* Write descriptor data to the bypass BAR */ + bypass_addr = (u32 *)xdev->bar[xdev->bypass_bar_idx]; + bypass_addr += engine->bypass_offset; + while (buf_offset < count) { + copy_err = copy_from_user(&desc_data, &buf[buf_offset], + sizeof(u32)); + if (!copy_err) { + write_register(desc_data, bypass_addr, bypass_addr - engine->bypass_offset); + buf_offset += sizeof(u32); + rc = buf_offset; + } else { + dbg_sg("Error reading data from userspace buffer\n"); + rc = -EINVAL; + break; + } + } + + spin_unlock(&engine->lock); + + + return rc; +} + + +/* + * character device file operations for bypass operation + */ + +static const struct file_operations bypass_fops = { + .owner = THIS_MODULE, + .open = char_open, + .release = char_close, + .read = char_bypass_read, + .write = char_bypass_write, + .mmap = bridge_mmap, +}; + +void cdev_bypass_init(struct xdma_cdev *xcdev) +{ + cdev_init(&xcdev->cdev, &bypass_fops); +} diff --git a/sources/xdma_driver/xdma/cdev_bypass.o.ur-safe b/sources/xdma_driver/xdma/cdev_bypass.o.ur-safe new file mode 100644 index 0000000..0cd6b8c --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_bypass.o.ur-safe @@ -0,0 +1,2 @@ +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/cdev_bypass.o-.text-23c +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/cdev_bypass.o-.text-e9 diff --git a/sources/xdma_driver/xdma/cdev_ctrl.c b/sources/xdma_driver/xdma/cdev_ctrl.c new file mode 100644 index 0000000..6699f5c --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_ctrl.c @@ -0,0 +1,267 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include <linux/ioctl.h> +#include "version.h" +#include "xdma_cdev.h" +#include "cdev_ctrl.h" + +/* + * character device file operations for control bus (through control bridge) + */ +static ssize_t char_ctrl_read(struct file *fp, char __user *buf, size_t count, + loff_t *pos) +{ + struct xdma_cdev *xcdev = (struct xdma_cdev *)fp->private_data; + struct xdma_dev *xdev; + void *reg; + u32 w; + int rv; + + rv = xcdev_check(__func__, xcdev, 0); + if (rv < 0) + return rv; + xdev = xcdev->xdev; + + /* only 32-bit aligned and 32-bit multiples */ + if (*pos & 3) + return -EPROTO; + /* first address is BAR base plus file position offset */ + reg = xdev->bar[xcdev->bar] + *pos; + //w = read_register(reg); + w = ioread32(reg); + dbg_sg("char_ctrl_read(@%p, count=%ld, pos=%d) value = 0x%08x\n", reg, + (long)count, (int)*pos, w); + rv = copy_to_user(buf, &w, 4); + if (rv) + dbg_sg("Copy to userspace failed but continuing\n"); + + *pos += 4; + return 4; +} + +static ssize_t char_ctrl_write(struct file *file, const char __user *buf, + size_t count, loff_t *pos) +{ + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + struct xdma_dev *xdev; + void *reg; + u32 w; + int rv; + + rv = xcdev_check(__func__, xcdev, 0); + if (rv < 0) + return rv; + xdev = xcdev->xdev; + + /* only 32-bit aligned and 32-bit multiples */ + if (*pos & 3) + return -EPROTO; + + /* first address is BAR base plus file position offset */ + reg = xdev->bar[xcdev->bar] + *pos; + rv = copy_from_user(&w, buf, 4); + if (rv) { + pr_info("copy from user failed %d/4, but continuing.\n", rv); + } + + dbg_sg("char_ctrl_write(0x%08x @%p, count=%ld, pos=%d)\n", w, reg, + (long)count, (int)*pos); + //write_register(w, reg); + iowrite32(w, reg); + *pos += 4; + return 4; +} + +static long version_ioctl(struct xdma_cdev *xcdev, void __user *arg) +{ + struct xdma_ioc_info obj; + struct xdma_dev *xdev = xcdev->xdev; + int rv; + + rv = copy_from_user((void *)&obj, arg, sizeof(struct xdma_ioc_info)); + if (rv) { + pr_info("copy from user failed %d/%ld.\n", + rv, sizeof(struct xdma_ioc_info)); + return -EFAULT; + } + memset(&obj, 0, sizeof(obj)); + obj.vendor = xdev->pdev->vendor; + obj.device = xdev->pdev->device; + obj.subsystem_vendor = xdev->pdev->subsystem_vendor; + obj.subsystem_device = xdev->pdev->subsystem_device; + obj.feature_id = xdev->feature_id; + obj.driver_version = DRV_MOD_VERSION_NUMBER; + obj.domain = 0; + obj.bus = PCI_BUS_NUM(xdev->pdev->devfn); + obj.dev = PCI_SLOT(xdev->pdev->devfn); + obj.func = PCI_FUNC(xdev->pdev->devfn); + if (copy_to_user(arg, &obj, sizeof(struct xdma_ioc_info))) + return -EFAULT; + return 0; +} + +long char_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct xdma_cdev *xcdev = (struct xdma_cdev *)filp->private_data; + struct xdma_dev *xdev; + struct xdma_ioc_base ioctl_obj; + long result = 0; + int rv; + + rv = xcdev_check(__func__, xcdev, 0); + if (rv < 0) + return rv; + xdev = xcdev->xdev; + + pr_info("cmd 0x%x, xdev 0x%p, pdev 0x%p.\n", cmd, xdev, xdev->pdev); + + if (_IOC_TYPE(cmd) != XDMA_IOC_MAGIC) { + pr_err("cmd %u, bad magic 0x%x/0x%x.\n", + cmd, _IOC_TYPE(cmd), XDMA_IOC_MAGIC); + return -ENOTTY; + } + + #if LINUX_VERSION_CODE < KERNEL_VERSION(5,3,0) + if (_IOC_DIR(cmd) & _IOC_READ) + result = !access_ok(VERIFY_WRITE, (void __user *)arg, + _IOC_SIZE(cmd)); + else if (_IOC_DIR(cmd) & _IOC_WRITE) + result = !access_ok(VERIFY_READ, (void __user *)arg, + _IOC_SIZE(cmd)); + #else + if (_IOC_DIR(cmd) & _IOC_READ) + result = !access_ok((void __user *)arg, + _IOC_SIZE(cmd)); + else if (_IOC_DIR(cmd) & _IOC_WRITE) + result = !access_ok((void __user *)arg, + _IOC_SIZE(cmd)); + #endif + + if (result) { + pr_err("bad access %ld.\n", result); + return -EFAULT; + } + + switch (cmd) { + case XDMA_IOCINFO: + if (copy_from_user((void *)&ioctl_obj, (void *) arg, + sizeof(struct xdma_ioc_base))) { + pr_err("copy_from_user failed.\n"); + return -EFAULT; + } + + if (ioctl_obj.magic != XDMA_XCL_MAGIC) { + pr_err("magic 0x%x != XDMA_XCL_MAGIC (0x%x).\n", + ioctl_obj.magic, XDMA_XCL_MAGIC); + return -ENOTTY; + } + + return version_ioctl(xcdev, (void __user *)arg); + case XDMA_IOCOFFLINE: + if (!xdev) { + pr_info("cmd %u, xdev NULL.\n", cmd); + return -EINVAL; + } + xdma_device_offline(xdev->pdev, xdev); + break; + case XDMA_IOCONLINE: + if (!xdev) { + pr_info("cmd %u, xdev NULL.\n", cmd); + return -EINVAL; + } + xdma_device_online(xdev->pdev, xdev); + break; + default: + pr_err("UNKNOWN ioctl cmd 0x%x.\n", cmd); + return -ENOTTY; + } + return 0; +} + +/* maps the PCIe BAR into user space for memory-like access using mmap() */ +int bridge_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct xdma_dev *xdev; + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + unsigned long off; + unsigned long phys; + unsigned long vsize; + unsigned long psize; + int rv; + + rv = xcdev_check(__func__, xcdev, 0); + if (rv < 0) + return rv; + xdev = xcdev->xdev; + + off = vma->vm_pgoff << PAGE_SHIFT; + /* BAR physical address */ + phys = pci_resource_start(xdev->pdev, xcdev->bar) + off; + vsize = vma->vm_end - vma->vm_start; + /* complete resource */ + psize = pci_resource_end(xdev->pdev, xcdev->bar) - + pci_resource_start(xdev->pdev, xcdev->bar) + 1 - off; + + dbg_sg("mmap(): xcdev = 0x%08lx\n", (unsigned long)xcdev); + dbg_sg("mmap(): cdev->bar = %d\n", xcdev->bar); + dbg_sg("mmap(): xdev = 0x%p\n", xdev); + dbg_sg("mmap(): pci_dev = 0x%08lx\n", (unsigned long)xdev->pdev); + + dbg_sg("off = 0x%lx\n", off); + dbg_sg("start = 0x%llx\n", + (unsigned long long)pci_resource_start(xdev->pdev, + xcdev->bar)); + dbg_sg("phys = 0x%lx\n", phys); + + if (vsize > psize) + return -EINVAL; + /* + * pages must not be cached as this would result in cache line sized + * accesses to the end point + */ + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + /* + * prevent touching the pages (byte access) for swap-in, + * and prevent the pages from being swapped out + */ + vma->vm_flags |= VMEM_FLAGS; + /* make MMIO accessible to user space */ + rv = io_remap_pfn_range(vma, vma->vm_start, phys >> PAGE_SHIFT, + vsize, vma->vm_page_prot); + dbg_sg("vma=0x%p, vma->vm_start=0x%lx, phys=0x%lx, size=%lu = %d\n", + vma, vma->vm_start, phys >> PAGE_SHIFT, vsize, rv); + + if (rv) + return -EAGAIN; + return 0; +} + +/* + * character device file operations for control bus (through control bridge) + */ +static const struct file_operations ctrl_fops = { + .owner = THIS_MODULE, + .open = char_open, + .release = char_close, + .read = char_ctrl_read, + .write = char_ctrl_write, + .mmap = bridge_mmap, + .unlocked_ioctl = char_ctrl_ioctl, +}; + +void cdev_ctrl_init(struct xdma_cdev *xcdev) +{ + cdev_init(&xcdev->cdev, &ctrl_fops); +} diff --git a/sources/xdma_driver/xdma/cdev_ctrl.h b/sources/xdma_driver/xdma/cdev_ctrl.h new file mode 100644 index 0000000..cb8ef6d --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_ctrl.h @@ -0,0 +1,80 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef _XDMA_IOCALLS_POSIX_H_ +#define _XDMA_IOCALLS_POSIX_H_ + +#include <linux/ioctl.h> + +/* Use 'x' as magic number */ +#define XDMA_IOC_MAGIC 'x' +/* XL OpenCL X->58(ASCII), L->6C(ASCII), O->0 C->C L->6C(ASCII); */ +#define XDMA_XCL_MAGIC 0X586C0C6C + +/* + * S means "Set" through a ptr, + * T means "Tell" directly with the argument value + * G means "Get": reply by setting through a pointer + * Q means "Query": response is on the return value + * X means "eXchange": switch G and S atomically + * H means "sHift": switch T and Q atomically + * + * _IO(type,nr) no arguments + * _IOR(type,nr,datatype) read data from driver + * _IOW(type,nr.datatype) write data to driver + * _IORW(type,nr,datatype) read/write data + * + * _IOC_DIR(nr) returns direction + * _IOC_TYPE(nr) returns magic + * _IOC_NR(nr) returns number + * _IOC_SIZE(nr) returns size + */ + +enum XDMA_IOC_TYPES { + XDMA_IOC_NOP, + XDMA_IOC_INFO, + XDMA_IOC_OFFLINE, + XDMA_IOC_ONLINE, + XDMA_IOC_MAX +}; + +struct xdma_ioc_base { + unsigned int magic; + unsigned int command; +}; + +struct xdma_ioc_info { + struct xdma_ioc_base base; + unsigned short vendor; + unsigned short device; + unsigned short subsystem_vendor; + unsigned short subsystem_device; + unsigned int dma_engine_version; + unsigned int driver_version; + unsigned long long feature_id; + unsigned short domain; + unsigned char bus; + unsigned char dev; + unsigned char func; +}; + +/* IOCTL codes */ +#define XDMA_IOCINFO _IOWR(XDMA_IOC_MAGIC, XDMA_IOC_INFO, \ + struct xdma_ioc_info) +#define XDMA_IOCOFFLINE _IO(XDMA_IOC_MAGIC, XDMA_IOC_OFFLINE) +#define XDMA_IOCONLINE _IO(XDMA_IOC_MAGIC, XDMA_IOC_ONLINE) + +#define IOCTL_XDMA_ADDRMODE_SET _IOW('q', 4, int) +#define IOCTL_XDMA_ADDRMODE_GET _IOR('q', 5, int) +#define IOCTL_XDMA_ALIGN_GET _IOR('q', 6, int) + +#endif /* _XDMA_IOCALLS_POSIX_H_ */ diff --git a/sources/xdma_driver/xdma/cdev_events.c b/sources/xdma_driver/xdma/cdev_events.c new file mode 100644 index 0000000..3ada7db --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_events.c @@ -0,0 +1,113 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include "xdma_cdev.h" + +/* + * character device file operations for events + */ +static ssize_t char_events_read(struct file *file, char __user *buf, + size_t count, loff_t *pos) +{ + int rv; + struct xdma_user_irq *user_irq; + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + u32 events_user; + unsigned long flags; + + rv = xcdev_check(__func__, xcdev, 0); + if (rv < 0) + return rv; + user_irq = xcdev->user_irq; + if (!user_irq) { + pr_info("xcdev 0x%p, user_irq NULL.\n", xcdev); + return -EINVAL; + } + + if (count != 4) + return -EPROTO; + + if (*pos & 3) + return -EPROTO; + + /* + * sleep until any interrupt events have occurred, + * or a signal arrived + */ + rv = wait_event_interruptible(user_irq->events_wq, + user_irq->events_irq != 0); + if (rv) + dbg_sg("wait_event_interruptible=%d\n", rv); + + /* wait_event_interruptible() was interrupted by a signal */ + if (rv == -ERESTARTSYS) + return -ERESTARTSYS; + + /* atomically decide which events are passed to the user */ + spin_lock_irqsave(&user_irq->events_lock, flags); + events_user = user_irq->events_irq; + user_irq->events_irq = 0; + spin_unlock_irqrestore(&user_irq->events_lock, flags); + + rv = copy_to_user(buf, &events_user, 4); + if (rv) + dbg_sg("Copy to user failed but continuing\n"); + + return 4; +} + +static unsigned int char_events_poll(struct file *file, poll_table *wait) +{ + struct xdma_user_irq *user_irq; + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + unsigned long flags; + unsigned int mask = 0; + int rv; + + rv = xcdev_check(__func__, xcdev, 0); + if (rv < 0) + return rv; + user_irq = xcdev->user_irq; + if (!user_irq) { + pr_info("xcdev 0x%p, user_irq NULL.\n", xcdev); + return -EINVAL; + } + + poll_wait(file, &user_irq->events_wq, wait); + + spin_lock_irqsave(&user_irq->events_lock, flags); + if (user_irq->events_irq) + mask = POLLIN | POLLRDNORM; /* readable */ + + spin_unlock_irqrestore(&user_irq->events_lock, flags); + + return mask; +} + +/* + * character device file operations for the irq events + */ +static const struct file_operations events_fops = { + .owner = THIS_MODULE, + .open = char_open, + .release = char_close, + .read = char_events_read, + .poll = char_events_poll, +}; + +void cdev_event_init(struct xdma_cdev *xcdev) +{ + xcdev->user_irq = &(xcdev->xdev->user_irq[xcdev->bar]); + cdev_init(&xcdev->cdev, &events_fops); +} diff --git a/sources/xdma_driver/xdma/cdev_sgdma.c b/sources/xdma_driver/xdma/cdev_sgdma.c new file mode 100644 index 0000000..410ef45 --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_sgdma.c @@ -0,0 +1,552 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include <asm/cacheflush.h> +#include "libxdma_api.h" +#include "xdma_cdev.h" +#include "cdev_sgdma.h" + +/* Module Parameters */ +unsigned int sgdma_timeout = 10; +module_param(sgdma_timeout, uint, 0644); +MODULE_PARM_DESC(sgdma_timeout, "timeout in seconds for sgdma, default is 10 sec."); + +/* + * character device file operations for SG DMA engine + */ +static loff_t char_sgdma_llseek(struct file *file, loff_t off, int whence) +{ + loff_t newpos = 0; + + switch (whence) { + case 0: /* SEEK_SET */ + newpos = off; + break; + case 1: /* SEEK_CUR */ + newpos = file->f_pos + off; + break; + case 2: /* SEEK_END, @TODO should work from end of address space */ + newpos = UINT_MAX + off; + break; + default: /* can't happen */ + return -EINVAL; + } + if (newpos < 0) + return -EINVAL; + file->f_pos = newpos; + dbg_fops("char_sgdma_llseek: pos=%lld\n", (signed long long)newpos); + +#if 0 + pr_err("0x%p, off 0x%lld, whence %d -> pos %lld.\n", + file, (signed long long)off, whence, (signed long long)off); +#endif + + return newpos; +} + +/* char_sgdma_read_write() -- Read from or write to the device + * + * @buf userspace buffer + * @count number of bytes in the userspace buffer + * @pos byte-address in device + * @dir_to_device If !0, a write to the device is performed + * + * Iterate over the userspace buffer, taking at most 255 * PAGE_SIZE bytes for + * each DMA transfer. + * + * For each transfer, get the user pages, build a sglist, map, build a + * descriptor table. submit the transfer. wait for the interrupt handler + * to wake us on completion. + */ + +static int check_transfer_align(struct xdma_engine *engine, + const char __user *buf, size_t count, loff_t pos, int sync) +{ + BUG_ON(!engine); + + /* AXI ST or AXI MM non-incremental addressing mode? */ + if (engine->non_incr_addr) { + int buf_lsb = (int)((uintptr_t)buf) & (engine->addr_align - 1); + size_t len_lsb = count & ((size_t)engine->len_granularity - 1); + int pos_lsb = (int)pos & (engine->addr_align - 1); + + dbg_tfr("AXI ST or MM non-incremental\n"); + dbg_tfr("buf_lsb = %d, pos_lsb = %d, len_lsb = %ld\n", buf_lsb, + pos_lsb, len_lsb); + + if (buf_lsb != 0) { + dbg_tfr("FAIL: non-aligned buffer address %p\n", buf); + return -EINVAL; + } + + if ((pos_lsb != 0) && (sync)) { + dbg_tfr("FAIL: non-aligned AXI MM FPGA addr 0x%llx\n", + (unsigned long long)pos); + return -EINVAL; + } + + if (len_lsb != 0) { + dbg_tfr("FAIL: len %d is not a multiple of %d\n", + (int)count, + (int)engine->len_granularity); + return -EINVAL; + } + /* AXI MM incremental addressing mode */ + } else { + int buf_lsb = (int)((uintptr_t)buf) & (engine->addr_align - 1); + int pos_lsb = (int)pos & (engine->addr_align - 1); + + if (buf_lsb != pos_lsb) { + dbg_tfr("FAIL: Misalignment error\n"); + dbg_tfr("host addr %p, FPGA addr 0x%llx\n", buf, pos); + return -EINVAL; + } + } + + return 0; +} + +/* + * Map a user memory range into a scatterlist + * inspired by vhost_scsi_map_to_sgl() + * Returns the number of scatterlist entries used or -errno on error. + */ +static inline void xdma_io_cb_release(struct xdma_io_cb *cb) +{ + int i; + + for (i = 0; i < cb->pages_nr; i++) + put_page(cb->pages[i]); + + sg_free_table(&cb->sgt); + kfree(cb->pages); + + memset(cb, 0, sizeof(*cb)); +} + +static void char_sgdma_unmap_user_buf(struct xdma_io_cb *cb, bool write) +{ + int i; + + sg_free_table(&cb->sgt); + + if (!cb->pages || !cb->pages_nr) + return; + + for (i = 0; i < cb->pages_nr; i++) { + if (cb->pages[i]) { + if (!write) + set_page_dirty_lock(cb->pages[i]); + put_page(cb->pages[i]); + } else + break; + } + + if (i != cb->pages_nr) + pr_info("sgl pages %d/%u.\n", i, cb->pages_nr); + + kfree(cb->pages); + cb->pages = NULL; +} + +static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write) +{ + struct sg_table *sgt = &cb->sgt; + unsigned long len = cb->len; + char *buf = cb->buf; + struct scatterlist *sg; + unsigned int pages_nr = (((unsigned long)buf + len + PAGE_SIZE -1) - + ((unsigned long)buf & PAGE_MASK)) + >> PAGE_SHIFT; + int i; + int rv; + + if (pages_nr == 0) { + return -EINVAL; + } + + if (sg_alloc_table(sgt, pages_nr, GFP_KERNEL)) { + pr_err("sgl OOM.\n"); + return -ENOMEM; + } + + cb->pages = kcalloc(pages_nr, sizeof(struct page *), GFP_KERNEL); + if (!cb->pages) { + pr_err("pages OOM.\n"); + rv = -ENOMEM; + goto err_out; + } + + rv = get_user_pages_fast((unsigned long)buf, pages_nr, 1/* write */, + cb->pages); + /* No pages were pinned */ + if (rv < 0) { + pr_err("unable to pin down %u user pages, %d.\n", + pages_nr, rv); + goto err_out; + } + /* Less pages pinned than wanted */ + if (rv != pages_nr) { + pr_err("unable to pin down all %u user pages, %d.\n", + pages_nr, rv); + rv = -EFAULT; + cb->pages_nr = rv; + goto err_out; + } + + for (i = 1; i < pages_nr; i++) { + if (cb->pages[i - 1] == cb->pages[i]) { + pr_err("duplicate pages, %d, %d.\n", + i - 1, i); + rv = -EFAULT; + cb->pages_nr = pages_nr; + goto err_out; + } + } + + sg = sgt->sgl; + for (i = 0; i < pages_nr; i++, sg = sg_next(sg)) { + //unsigned int offset = (uintptr_t)buf & ~PAGE_MASK; + unsigned int offset = offset_in_page(buf); + unsigned int nbytes = min_t(unsigned int, PAGE_SIZE - offset, len); + + flush_dcache_page(cb->pages[i]); + sg_set_page(sg, cb->pages[i], nbytes, offset); + + buf += nbytes; + len -= nbytes; + } + + BUG_ON(len); + cb->pages_nr = pages_nr; + return 0; + +err_out: + char_sgdma_unmap_user_buf(cb, write); + + return rv; +} + +static ssize_t char_sgdma_read_write(struct file *file, char __user *buf, + size_t count, loff_t *pos, bool write) +{ + int rv; + ssize_t res = 0; + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + struct xdma_dev *xdev; + struct xdma_engine *engine; + struct xdma_io_cb cb; + + rv = xcdev_check(__func__, xcdev, 1); + if (rv < 0) + return rv; + xdev = xcdev->xdev; + engine = xcdev->engine; + + dbg_tfr("file 0x%p, priv 0x%p, buf 0x%p,%llu, pos %llu, W %d, %s.\n", + file, file->private_data, buf, (u64)count, (u64)*pos, write, + engine->name); + + if ((write && engine->dir != DMA_TO_DEVICE) || + (!write && engine->dir != DMA_FROM_DEVICE)) { + pr_err("r/w mismatch. W %d, dir %d.\n", + write, engine->dir); + return -EINVAL; + } + + rv = check_transfer_align(engine, buf, count, *pos, 1); + if (rv) { + pr_info("Invalid transfer alignment detected\n"); + return rv; + } + + memset(&cb, 0, sizeof(struct xdma_io_cb)); + cb.buf = buf; + cb.len = count; + rv = char_sgdma_map_user_buf_to_sgl(&cb, write); + if (rv < 0) + return rv; + + res = xdma_xfer_submit(xdev, engine->channel, write, *pos, &cb.sgt, + 0, sgdma_timeout * 1000); + //pr_err("xfer_submit return=%lld.\n", (s64)res); + + //interrupt_status(xdev); + + char_sgdma_unmap_user_buf(&cb, write); + + return res; +} + + +static ssize_t char_sgdma_write(struct file *file, const char __user *buf, + size_t count, loff_t *pos) +{ + return char_sgdma_read_write(file, (char *)buf, count, pos, 1); +} + +static ssize_t char_sgdma_read(struct file *file, char __user *buf, + size_t count, loff_t *pos) +{ + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + struct xdma_engine *engine; + int rv; + + rv = xcdev_check(__func__, xcdev, 1); + if (rv < 0) + return rv; + + engine = xcdev->engine; + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { + rv = xdma_cyclic_transfer_setup(engine); + if (rv < 0 && rv != -EBUSY) + return rv; + /* 600 sec. timeout */ + return xdma_engine_read_cyclic(engine, buf, count, 600000); + } + + return char_sgdma_read_write(file, (char *)buf, count, pos, 0); +} + +static int ioctl_do_perf_start(struct xdma_engine *engine, unsigned long arg) +{ + int rv; + struct xdma_dev *xdev; + + BUG_ON(!engine); + xdev = engine->xdev; + BUG_ON(!xdev); + + /* performance measurement already running on this engine? */ + if (engine->xdma_perf) { + dbg_perf("IOCTL_XDMA_PERF_START failed!\n"); + dbg_perf("Perf measurement already seems to be running!\n"); + return -EBUSY; + } + engine->xdma_perf = kzalloc(sizeof(struct xdma_performance_ioctl), + GFP_KERNEL); + + if (!engine->xdma_perf) + return -ENOMEM; + + rv = copy_from_user(engine->xdma_perf, + (struct xdma_performance_ioctl *)arg, + sizeof(struct xdma_performance_ioctl)); + + if (rv < 0) { + dbg_perf("Failed to copy from user space 0x%lx\n", arg); + return -EINVAL; + } + if (engine->xdma_perf->version != IOCTL_XDMA_PERF_V1) { + dbg_perf("Unsupported IOCTL version %d\n", + engine->xdma_perf->version); + return -EINVAL; + } + + enable_perf(engine); + dbg_perf("transfer_size = %d\n", engine->xdma_perf->transfer_size); + /* initialize wait queue */ + init_waitqueue_head(&engine->xdma_perf_wq); + xdma_performance_submit(xdev, engine); + + return 0; +} + +static int ioctl_do_perf_stop(struct xdma_engine *engine, unsigned long arg) +{ + struct xdma_transfer *transfer = NULL; + int rv; + + dbg_perf("IOCTL_XDMA_PERF_STOP\n"); + + /* no performance measurement running on this engine? */ + if (!engine->xdma_perf) { + dbg_perf("No measurement in progress\n"); + return -EINVAL; + } + + /* stop measurement */ + transfer = engine_cyclic_stop(engine); + dbg_perf("Waiting for measurement to stop\n"); + + if (engine->xdma_perf) { + get_perf_stats(engine); + + rv = copy_to_user((void __user *)arg, engine->xdma_perf, + sizeof(struct xdma_performance_ioctl)); + if (rv) { + dbg_perf("Error copying result to user\n"); + return -EINVAL; + } + } else { + dbg_perf("engine->xdma_perf == NULL?\n"); + } + + kfree(engine->xdma_perf); + engine->xdma_perf = NULL; + + return 0; +} + +static int ioctl_do_perf_get(struct xdma_engine *engine, unsigned long arg) +{ + int rc; + + BUG_ON(!engine); + + dbg_perf("IOCTL_XDMA_PERF_GET\n"); + + if (engine->xdma_perf) { + get_perf_stats(engine); + + rc = copy_to_user((void __user *)arg, engine->xdma_perf, + sizeof(struct xdma_performance_ioctl)); + if (rc) { + dbg_perf("Error copying result to user\n"); + return -EINVAL; + } + } else { + dbg_perf("engine->xdma_perf == NULL?\n"); + return -EPROTO; + } + + return 0; +} + +static int ioctl_do_addrmode_set(struct xdma_engine *engine, unsigned long arg) +{ + return engine_addrmode_set(engine, arg); +} + +static int ioctl_do_addrmode_get(struct xdma_engine *engine, unsigned long arg) +{ + int rv; + unsigned long src; + + BUG_ON(!engine); + src = !!engine->non_incr_addr; + + dbg_perf("IOCTL_XDMA_ADDRMODE_GET\n"); + rv = put_user(src, (int __user *)arg); + + return rv; +} + +static int ioctl_do_align_get(struct xdma_engine *engine, unsigned long arg) +{ + BUG_ON(!engine); + + dbg_perf("IOCTL_XDMA_ALIGN_GET\n"); + return put_user(engine->addr_align, (int __user *)arg); +} + +static long char_sgdma_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + struct xdma_dev *xdev; + struct xdma_engine *engine; + + int rv = 0; + + rv = xcdev_check(__func__, xcdev, 1); + if (rv < 0) + return rv; + + xdev = xcdev->xdev; + engine = xcdev->engine; + + switch (cmd) { + case IOCTL_XDMA_PERF_START: + rv = ioctl_do_perf_start(engine, arg); + break; + case IOCTL_XDMA_PERF_STOP: + rv = ioctl_do_perf_stop(engine, arg); + break; + case IOCTL_XDMA_PERF_GET: + rv = ioctl_do_perf_get(engine, arg); + break; + case IOCTL_XDMA_ADDRMODE_SET: + rv = ioctl_do_addrmode_set(engine, arg); + break; + case IOCTL_XDMA_ADDRMODE_GET: + rv = ioctl_do_addrmode_get(engine, arg); + break; + case IOCTL_XDMA_ALIGN_GET: + rv = ioctl_do_align_get(engine, arg); + break; + default: + dbg_perf("Unsupported operation\n"); + rv = -EINVAL; + break; + } + + return rv; +} + +static int char_sgdma_open(struct inode *inode, struct file *file) +{ + struct xdma_cdev *xcdev; + struct xdma_engine *engine; + + char_open(inode, file); + + xcdev = (struct xdma_cdev *)file->private_data; + engine = xcdev->engine; + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { + if (engine->device_open == 1) + return -EBUSY; + else + engine->device_open = 1; + } + + return 0; +} + +static int char_sgdma_close(struct inode *inode, struct file *file) +{ + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + struct xdma_engine *engine; + int rv; + + rv = xcdev_check(__func__, xcdev, 1); + if (rv < 0) + return rv; + + engine = xcdev->engine; + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { + engine->device_open = 0; + if (engine->cyclic_req) + return xdma_cyclic_transfer_teardown(engine); + } + + return 0; +} +static const struct file_operations sgdma_fops = { + .owner = THIS_MODULE, + .open = char_sgdma_open, + .release = char_sgdma_close, + .write = char_sgdma_write, + .read = char_sgdma_read, + .unlocked_ioctl = char_sgdma_ioctl, + .llseek = char_sgdma_llseek, +}; + +void cdev_sgdma_init(struct xdma_cdev *xcdev) +{ + cdev_init(&xcdev->cdev, &sgdma_fops); +} diff --git a/sources/xdma_driver/xdma/cdev_sgdma.h b/sources/xdma_driver/xdma/cdev_sgdma.h new file mode 100644 index 0000000..7781fae --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_sgdma.h @@ -0,0 +1,66 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef _XDMA_IOCALLS_POSIX_H_ +#define _XDMA_IOCALLS_POSIX_H_ + +#include <linux/ioctl.h> + + +#define IOCTL_XDMA_PERF_V1 (1) +#define XDMA_ADDRMODE_MEMORY (0) +#define XDMA_ADDRMODE_FIXED (1) + +/* + * S means "Set" through a ptr, + * T means "Tell" directly with the argument value + * G means "Get": reply by setting through a pointer + * Q means "Query": response is on the return value + * X means "eXchange": switch G and S atomically + * H means "sHift": switch T and Q atomically + * + * _IO(type,nr) no arguments + * _IOR(type,nr,datatype) read data from driver + * _IOW(type,nr.datatype) write data to driver + * _IORW(type,nr,datatype) read/write data + * + * _IOC_DIR(nr) returns direction + * _IOC_TYPE(nr) returns magic + * _IOC_NR(nr) returns number + * _IOC_SIZE(nr) returns size + */ + +struct xdma_performance_ioctl +{ + /* IOCTL_XDMA_IOCTL_Vx */ + uint32_t version; + uint32_t transfer_size; + /* measurement */ + uint32_t stopped; + uint32_t iterations; + uint64_t clock_cycle_count; + uint64_t data_cycle_count; + uint64_t pending_count; +}; + + + +/* IOCTL codes */ + +#define IOCTL_XDMA_PERF_START _IOW('q', 1, struct xdma_performance_ioctl *) +#define IOCTL_XDMA_PERF_STOP _IOW('q', 2, struct xdma_performance_ioctl *) +#define IOCTL_XDMA_PERF_GET _IOR('q', 3, struct xdma_performance_ioctl *) +#define IOCTL_XDMA_ADDRMODE_SET _IOW('q', 4, int) +#define IOCTL_XDMA_ADDRMODE_GET _IOR('q', 5, int) +#define IOCTL_XDMA_ALIGN_GET _IOR('q', 6, int) + +#endif /* _XDMA_IOCALLS_POSIX_H_ */ diff --git a/sources/xdma_driver/xdma/cdev_xvc.c b/sources/xdma_driver/xdma/cdev_xvc.c new file mode 100644 index 0000000..a077bb1 --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_xvc.c @@ -0,0 +1,238 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include "xdma_cdev.h" +#include "cdev_xvc.h" + +#define COMPLETION_LOOP_MAX 100 + +#define XVC_BAR_LENGTH_REG 0x0 +#define XVC_BAR_TMS_REG 0x4 +#define XVC_BAR_TDI_REG 0x8 +#define XVC_BAR_TDO_REG 0xC +#define XVC_BAR_CTRL_REG 0x10 + +#ifdef __REG_DEBUG__ +/* SECTION: Function definitions */ +inline void __write_register(const char *fn, u32 value, void *base, + unsigned int off) +{ + pr_info("%s: 0x%p, W reg 0x%lx, 0x%x.\n", fn, base, off, value); + iowrite32(value, base + off); +} + +inline u32 __read_register(const char *fn, void *base, unsigned int off) +{ + u32 v = ioread32(base + off); + + pr_info("%s: 0x%p, R reg 0x%lx, 0x%x.\n", fn, base, off, v); + return v; +} +#define write_register(v,base,off) __write_register(__func__, v, base, off) +#define read_register(base,off) __read_register(__func__, base, off) + +#else +#define write_register(v,base,off) iowrite32(v, (base) + (off)) +#define read_register(base,off) ioread32((base) + (off)) +#endif /* #ifdef __REG_DEBUG__ */ + + +static int xvc_shift_bits(void *base, u32 tms_bits, u32 tdi_bits, + u32 *tdo_bits) +{ + u32 control; + int count; + + /* set tms bit */ + write_register(tms_bits, base, XVC_BAR_TMS_REG); + /* set tdi bits and shift data out */ + write_register(tdi_bits, base, XVC_BAR_TDI_REG); + /* enable shift operation */ + write_register(0x1, base, XVC_BAR_CTRL_REG); + + /* poll for completion */ + count = COMPLETION_LOOP_MAX; + while (count) { + /* read control reg to check shift operation completion */ + control = read_register(base, XVC_BAR_CTRL_REG); + if ((control & 0x01) == 0) + break; + + count--; + } + + if (!count) { + pr_warn("XVC bar transaction timed out (0x%0X)\n", control); + return -ETIMEDOUT; + } + + /* read tdo bits back out */ + *tdo_bits = read_register(base, XVC_BAR_TDO_REG); + + return 0; +} + +static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct xdma_cdev *xcdev = (struct xdma_cdev *)filp->private_data; + struct xdma_dev *xdev; + struct xvc_ioc xvc_obj; + unsigned int opcode; + unsigned int total_bits; + unsigned int total_bytes; + unsigned char *buffer = NULL; + unsigned char *tms_buf = NULL; + unsigned char *tdi_buf = NULL; + unsigned char *tdo_buf = NULL; + unsigned int bits, bits_left; + void __iomem *iobase; + int rv; + + rv = xcdev_check(__func__, xcdev, 0); + if (rv < 0) + return rv; + xdev = xcdev->xdev; + + if (cmd != XDMA_IOCXVC) { + pr_info("ioctl 0x%x, UNKNOWN cmd.\n", cmd); + return -ENOIOCTLCMD; + } + + rv = copy_from_user((void *)&xvc_obj, (void __user *)arg, + sizeof(struct xvc_ioc)); + /* anything not copied ? */ + if (rv) { + pr_info("copy_from_user xvc_obj failed: %d.\n", rv); + goto cleanup; + } + + opcode = xvc_obj.opcode; + + /* Invalid operation type, no operation performed */ + if (opcode != 0x01 && opcode != 0x02) { + pr_info("UNKNOWN opcode 0x%x.\n", opcode); + return -EINVAL; + } + + total_bits = xvc_obj.length; + total_bytes = (total_bits + 7) >> 3; + + buffer = (char *)kmalloc(total_bytes * 3, GFP_KERNEL); + if (!buffer) { + pr_info("OOM %u, op 0x%x, len %u bits, %u bytes.\n", + 3 * total_bytes, opcode, total_bits, total_bytes); + rv = -ENOMEM; + goto cleanup; + } + tms_buf = buffer; + tdi_buf = tms_buf + total_bytes; + tdo_buf = tdi_buf + total_bytes; + + rv = copy_from_user((void *)tms_buf, xvc_obj.tms_buf, total_bytes); + if (rv) { + pr_info("copy tmfs_buf failed: %d/%u.\n", rv, total_bytes); + goto cleanup; + } + rv = copy_from_user((void *)tdi_buf, xvc_obj.tdi_buf, total_bytes); + if (rv) { + pr_info("copy tdi_buf failed: %d/%u.\n", rv, total_bytes); + goto cleanup; + } + + /* exclusive access */ + spin_lock(&xcdev->lock); + + iobase = xdev->bar[xcdev->bar] + xcdev->base; + /* set length register to 32 initially if more than one + * word-transaction is to be done */ + if (total_bits >= 32) + write_register(0x20, iobase, XVC_BAR_LENGTH_REG); + + for (bits = 0, bits_left = total_bits; bits < total_bits; bits += 32, + bits_left -= 32) { + unsigned int bytes = bits >> 3; + unsigned int shift_bytes = 4; + u32 tms_store = 0; + u32 tdi_store = 0; + u32 tdo_store = 0; + + if (bits_left < 32) { + /* set number of bits to shift out */ + write_register(bits_left, iobase, XVC_BAR_LENGTH_REG); + shift_bytes = (bits_left + 7) >> 3; + } + + memcpy(&tms_store, tms_buf + bytes, shift_bytes); + memcpy(&tdi_store, tdi_buf + bytes, shift_bytes); + + /* Shift data out and copy to output buffer */ + rv = xvc_shift_bits(iobase, tms_store, tdi_store, &tdo_store); + if (rv < 0) + goto cleanup; + + memcpy(tdo_buf + bytes, &tdo_store, shift_bytes); + } + + /* if testing bar access swap tdi and tdo bufferes to "loopback" */ + if (opcode == 0x2) { + char *tmp = tdo_buf; + + tdo_buf = tdi_buf; + tdi_buf = tmp; + } + + rv = copy_to_user((void *)xvc_obj.tdo_buf, tdo_buf, total_bytes); + if (rv) { + pr_info("copy back tdo_buf failed: %d/%u.\n", rv, total_bytes); + rv = -EFAULT; + goto cleanup; + } + +cleanup: + if (buffer) + kfree(buffer); + + #if LINUX_VERSION_CODE < KERNEL_VERSION(5,3,0) + mmiowb(); + #endif + + spin_unlock(&xcdev->lock); + + return rv; +} + +/* + * character device file operations for the XVC + */ +static const struct file_operations xvc_fops = { + .owner = THIS_MODULE, + .open = char_open, + .release = char_close, + .unlocked_ioctl = xvc_ioctl, +}; + +void cdev_xvc_init(struct xdma_cdev *xcdev) +{ +#ifdef __XVC_BAR_NUM__ + xcdev->bar = __XVC_BAR_NUM__; +#endif +#ifdef __XVC_BAR_OFFSET__ + xcdev->base = __XVC_BAR_OFFSET__; +#else + xcdev->base = XVC_BAR_OFFSET_DFLT; +#endif + pr_info("xcdev 0x%p, bar %u, offset 0x%lx.\n", + xcdev, xcdev->bar, xcdev->base); + cdev_init(&xcdev->cdev, &xvc_fops); +} diff --git a/sources/xdma_driver/xdma/cdev_xvc.h b/sources/xdma_driver/xdma/cdev_xvc.h new file mode 100644 index 0000000..e689706 --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_xvc.h @@ -0,0 +1,36 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef __XVC_IOCTL_H__ +#define __XVC_IOCTL_H__ + +#include <linux/ioctl.h> + +/* + * !!! TODO !!! + * need a better way set the bar offset dynamicly + */ +#define XVC_BAR_OFFSET_DFLT 0x40000 /* DSA 4.0 */ + +#define XVC_MAGIC 0x58564344 // "XVCD" + +struct xvc_ioc { + unsigned int opcode; + unsigned int length; + unsigned char *tms_buf; + unsigned char *tdi_buf; + unsigned char *tdo_buf; +}; + +#define XDMA_IOCXVC _IOWR(XVC_MAGIC, 1, struct xvc_ioc) + +#endif /* __XVC_IOCTL_H__ */ diff --git a/sources/xdma_driver/xdma/cdev_xvc.o.ur-safe b/sources/xdma_driver/xdma/cdev_xvc.o.ur-safe new file mode 100644 index 0000000..3a9a009 --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_xvc.o.ur-safe @@ -0,0 +1 @@ +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/cdev_xvc.o-.text-315 diff --git a/sources/xdma_driver/xdma/libxdma.c b/sources/xdma_driver/xdma/libxdma.c new file mode 100644 index 0000000..1743780 --- /dev/null +++ b/sources/xdma_driver/xdma/libxdma.c @@ -0,0 +1,4445 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/string.h> +#include <linux/mm.h> +#include <linux/errno.h> +#include <linux/sched.h> +#include <linux/vmalloc.h> + +#include "libxdma.h" +#include "libxdma_api.h" +#include "cdev_sgdma.h" + +/* SECTION: Module licensing */ + +#ifdef __LIBXDMA_MOD__ +#include "version.h" +#define DRV_MODULE_NAME "libxdma" +#define DRV_MODULE_DESC "Xilinx XDMA Base Driver" +#define DRV_MODULE_RELDATE "Feb. 2018" + +static char version[] = + DRV_MODULE_DESC " " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n"; + +MODULE_AUTHOR("Xilinx, Inc."); +MODULE_DESCRIPTION(DRV_MODULE_DESC); +MODULE_VERSION(DRV_MODULE_VERSION); +MODULE_LICENSE("Dual BSD/GPL"); +#endif + +/* Module Parameters */ +static unsigned int poll_mode; +module_param(poll_mode, uint, 0644); +MODULE_PARM_DESC(poll_mode, "Set 1 for hw polling, default is 0 (interrupts)"); + +static unsigned int interrupt_mode; +module_param(interrupt_mode, uint, 0644); +MODULE_PARM_DESC(interrupt_mode, "0 - MSI-x , 1 - MSI, 2 - Legacy"); + +static unsigned int enable_credit_mp; +module_param(enable_credit_mp, uint, 0644); +MODULE_PARM_DESC(enable_credit_mp, "Set 1 to enable creidt feature, default is 0 (no credit control)"); + +unsigned int desc_blen_max = XDMA_DESC_BLEN_MAX; +module_param(desc_blen_max, uint, 0644); +MODULE_PARM_DESC(desc_blen_max, "per descriptor max. buffer length, default is (1 << 28) - 1"); + +/* + * xdma device management + * maintains a list of the xdma devices + */ +static LIST_HEAD(xdev_list); +static DEFINE_MUTEX(xdev_mutex); + +static LIST_HEAD(xdev_rcu_list); +static DEFINE_SPINLOCK(xdev_rcu_lock); + +#ifndef list_last_entry +#define list_last_entry(ptr, type, member) \ + list_entry((ptr)->prev, type, member) +#endif + +static inline void xdev_list_add(struct xdma_dev *xdev) +{ + mutex_lock(&xdev_mutex); + if (list_empty(&xdev_list)) + xdev->idx = 0; + else { + struct xdma_dev *last; + + last = list_last_entry(&xdev_list, struct xdma_dev, list_head); + xdev->idx = last->idx + 1; + } + list_add_tail(&xdev->list_head, &xdev_list); + mutex_unlock(&xdev_mutex); + + dbg_init("dev %s, xdev 0x%p, xdma idx %d.\n", + dev_name(&xdev->pdev->dev), xdev, xdev->idx); + + spin_lock(&xdev_rcu_lock); + list_add_tail_rcu(&xdev->rcu_node, &xdev_rcu_list); + spin_unlock(&xdev_rcu_lock); +} + +#undef list_last_entry + +static inline void xdev_list_remove(struct xdma_dev *xdev) +{ + mutex_lock(&xdev_mutex); + list_del(&xdev->list_head); + mutex_unlock(&xdev_mutex); + + spin_lock(&xdev_rcu_lock); + list_del_rcu(&xdev->rcu_node); + spin_unlock(&xdev_rcu_lock); + synchronize_rcu(); +} + +struct xdma_dev *xdev_find_by_pdev(struct pci_dev *pdev) +{ + struct xdma_dev *xdev, *tmp; + + mutex_lock(&xdev_mutex); + list_for_each_entry_safe(xdev, tmp, &xdev_list, list_head) { + if (xdev->pdev == pdev) { + mutex_unlock(&xdev_mutex); + return xdev; + } + } + mutex_unlock(&xdev_mutex); + return NULL; +} +EXPORT_SYMBOL_GPL(xdev_find_by_pdev); + +static inline int debug_check_dev_hndl(const char *fname, struct pci_dev *pdev, + void *hndl) +{ + struct xdma_dev *xdev; + + if (!pdev) + return -EINVAL; + + xdev = xdev_find_by_pdev(pdev); + if (!xdev) { + pr_info("%s pdev 0x%p, hndl 0x%p, NO match found!\n", + fname, pdev, hndl); + return -EINVAL; + } + if (xdev != hndl) { + pr_err("%s pdev 0x%p, hndl 0x%p != 0x%p!\n", + fname, pdev, hndl, xdev); + return -EINVAL; + } + + return 0; +} + +#ifdef __LIBXDMA_DEBUG__ +/* SECTION: Function definitions */ +inline void __write_register(const char *fn, u32 value, void *iomem, unsigned long off) +{ + pr_err("%s: w reg 0x%lx(0x%p), 0x%x.\n", fn, off, iomem, value); + iowrite32(value, iomem); +} +#define write_register(v,mem,off) __write_register(__func__, v, mem, off) +#else +#define write_register(v,mem,off) iowrite32(v, mem) +#endif + +inline u32 read_register(void *iomem) +{ + return ioread32(iomem); +} + +static inline u32 build_u32(u32 hi, u32 lo) +{ + return ((hi & 0xFFFFUL) << 16) | (lo & 0xFFFFUL); +} + +static inline u64 build_u64(u64 hi, u64 lo) +{ + return ((hi & 0xFFFFFFFULL) << 32) | (lo & 0xFFFFFFFFULL); +} + +static void check_nonzero_interrupt_status(struct xdma_dev *xdev) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + u32 w; + + w = read_register(®->user_int_enable); + if (w) + pr_info("%s xdma%d user_int_enable = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + + w = read_register(®->channel_int_enable); + if (w) + pr_info("%s xdma%d channel_int_enable = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + + w = read_register(®->user_int_request); + if (w) + pr_info("%s xdma%d user_int_request = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + w = read_register(®->channel_int_request); + if (w) + pr_info("%s xdma%d channel_int_request = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + + w = read_register(®->user_int_pending); + if (w) + pr_info("%s xdma%d user_int_pending = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + w = read_register(®->channel_int_pending); + if (w) + pr_info("%s xdma%d channel_int_pending = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); +} + +/* channel_interrupts_enable -- Enable interrupts we are interested in */ +static void channel_interrupts_enable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->channel_int_enable_w1s, XDMA_OFS_INT_CTRL); +} + +/* channel_interrupts_disable -- Disable interrupts we not interested in */ +static void channel_interrupts_disable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->channel_int_enable_w1c, XDMA_OFS_INT_CTRL); +} + +/* user_interrupts_enable -- Enable interrupts we are interested in */ +static void user_interrupts_enable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->user_int_enable_w1s, XDMA_OFS_INT_CTRL); +} + +/* user_interrupts_disable -- Disable interrupts we not interested in */ +static void user_interrupts_disable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->user_int_enable_w1c, XDMA_OFS_INT_CTRL); +} + +/* read_interrupts -- Print the interrupt controller status */ +static u32 read_interrupts(struct xdma_dev *xdev) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + u32 lo; + u32 hi; + + /* extra debugging; inspect complete engine set of registers */ + hi = read_register(®->user_int_request); + dbg_io("ioread32(0x%p) returned 0x%08x (user_int_request).\n", + ®->user_int_request, hi); + lo = read_register(®->channel_int_request); + dbg_io("ioread32(0x%p) returned 0x%08x (channel_int_request)\n", + ®->channel_int_request, lo); + + /* return interrupts: user in upper 16-bits, channel in lower 16-bits */ + return build_u32(hi, lo); +} + +void enable_perf(struct xdma_engine *engine) +{ + u32 w; + + w = XDMA_PERF_CLEAR; + write_register(w, &engine->regs->perf_ctrl, + (unsigned long)(&engine->regs->perf_ctrl) - + (unsigned long)(&engine->regs)); + read_register(&engine->regs->identifier); + w = XDMA_PERF_AUTO | XDMA_PERF_RUN; + write_register(w, &engine->regs->perf_ctrl, + (unsigned long)(&engine->regs->perf_ctrl) - + (unsigned long)(&engine->regs)); + read_register(&engine->regs->identifier); + + dbg_perf("IOCTL_XDMA_PERF_START\n"); + +} +EXPORT_SYMBOL_GPL(enable_perf); + +void get_perf_stats(struct xdma_engine *engine) +{ + u32 hi; + u32 lo; + + BUG_ON(!engine); + + if (!engine->xdma_perf) { + pr_info("%s perf struct not set up.\n", engine->name); + return; + } + + hi = 0; + lo = read_register(&engine->regs->completed_desc_count); + engine->xdma_perf->iterations = build_u64(hi, lo); + + hi = read_register(&engine->regs->perf_cyc_hi); + lo = read_register(&engine->regs->perf_cyc_lo); + + engine->xdma_perf->clock_cycle_count = build_u64(hi, lo); + + hi = read_register(&engine->regs->perf_dat_hi); + lo = read_register(&engine->regs->perf_dat_lo); + engine->xdma_perf->data_cycle_count = build_u64(hi, lo); + + hi = read_register(&engine->regs->perf_pnd_hi); + lo = read_register(&engine->regs->perf_pnd_lo); + engine->xdma_perf->pending_count = build_u64(hi, lo); +} +EXPORT_SYMBOL_GPL(get_perf_stats); + +static void engine_reg_dump(struct xdma_engine *engine) +{ + u32 w; + + BUG_ON(!engine); + + w = read_register(&engine->regs->identifier); + pr_info("%s: ioread32(0x%p) = 0x%08x (id).\n", + engine->name, &engine->regs->identifier, w); + w &= BLOCK_ID_MASK; + if (w != BLOCK_ID_HEAD) { + pr_info("%s: engine id missing, 0x%08x exp. & 0x%x = 0x%x\n", + engine->name, w, BLOCK_ID_MASK, BLOCK_ID_HEAD); + return; + } + /* extra debugging; inspect complete engine set of registers */ + w = read_register(&engine->regs->status); + pr_info("%s: ioread32(0x%p) = 0x%08x (status).\n", + engine->name, &engine->regs->status, w); + w = read_register(&engine->regs->control); + pr_info("%s: ioread32(0x%p) = 0x%08x (control)\n", + engine->name, &engine->regs->control, w); + w = read_register(&engine->sgdma_regs->first_desc_lo); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_lo)\n", + engine->name, &engine->sgdma_regs->first_desc_lo, w); + w = read_register(&engine->sgdma_regs->first_desc_hi); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_hi)\n", + engine->name, &engine->sgdma_regs->first_desc_hi, w); + w = read_register(&engine->sgdma_regs->first_desc_adjacent); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_adjacent).\n", + engine->name, &engine->sgdma_regs->first_desc_adjacent, w); + w = read_register(&engine->regs->completed_desc_count); + pr_info("%s: ioread32(0x%p) = 0x%08x (completed_desc_count).\n", + engine->name, &engine->regs->completed_desc_count, w); + w = read_register(&engine->regs->interrupt_enable_mask); + pr_info("%s: ioread32(0x%p) = 0x%08x (interrupt_enable_mask)\n", + engine->name, &engine->regs->interrupt_enable_mask, w); +} + +/** + * engine_status_read() - read status of SG DMA engine (optionally reset) + * + * Stores status in engine->status. + * + * @return -1 on failure, status register otherwise + */ +static void engine_status_dump(struct xdma_engine *engine) +{ + u32 v = engine->status; + char buffer[256]; + char *buf = buffer; + int len = 0; + + len = sprintf(buf, "SG engine %s status: 0x%08x: ", engine->name, v); + + if ((v & XDMA_STAT_BUSY)) + len += sprintf(buf + len, "BUSY,"); + if ((v & XDMA_STAT_DESC_STOPPED)) + len += sprintf(buf + len, "DESC_STOPPED,"); + if ((v & XDMA_STAT_DESC_COMPLETED)) + len += sprintf(buf + len, "DESC_COMPL,"); + + /* common H2C & C2H */ + if ((v & XDMA_STAT_COMMON_ERR_MASK)) { + if ((v & XDMA_STAT_ALIGN_MISMATCH)) + len += sprintf(buf + len, "ALIGN_MISMATCH "); + if ((v & XDMA_STAT_MAGIC_STOPPED)) + len += sprintf(buf + len, "MAGIC_STOPPED "); + if ((v & XDMA_STAT_INVALID_LEN)) + len += sprintf(buf + len, "INVLIAD_LEN "); + if ((v & XDMA_STAT_IDLE_STOPPED)) + len += sprintf(buf + len, "IDLE_STOPPED "); + buf[len - 1] = ','; + } + + if ((engine->dir == DMA_TO_DEVICE)) { + /* H2C only */ + if ((v & XDMA_STAT_H2C_R_ERR_MASK)) { + len += sprintf(buf + len, "R:"); + if ((v & XDMA_STAT_H2C_R_UNSUPP_REQ)) + len += sprintf(buf + len, "UNSUPP_REQ "); + if ((v & XDMA_STAT_H2C_R_COMPL_ABORT)) + len += sprintf(buf + len, "COMPL_ABORT "); + if ((v & XDMA_STAT_H2C_R_PARITY_ERR)) + len += sprintf(buf + len, "PARITY "); + if ((v & XDMA_STAT_H2C_R_HEADER_EP)) + len += sprintf(buf + len, "HEADER_EP "); + if ((v & XDMA_STAT_H2C_R_UNEXP_COMPL)) + len += sprintf(buf + len, "UNEXP_COMPL "); + buf[len - 1] = ','; + } + + if ((v & XDMA_STAT_H2C_W_ERR_MASK)) { + len += sprintf(buf + len, "W:"); + if ((v & XDMA_STAT_H2C_W_DECODE_ERR)) + len += sprintf(buf + len, "DECODE_ERR "); + if ((v & XDMA_STAT_H2C_W_SLAVE_ERR)) + len += sprintf(buf + len, "SLAVE_ERR "); + buf[len - 1] = ','; + } + + } else { + /* C2H only */ + if ((v & XDMA_STAT_C2H_R_ERR_MASK)) { + len += sprintf(buf + len, "R:"); + if ((v & XDMA_STAT_C2H_R_DECODE_ERR)) + len += sprintf(buf + len, "DECODE_ERR "); + if ((v & XDMA_STAT_C2H_R_SLAVE_ERR)) + len += sprintf(buf + len, "SLAVE_ERR "); + buf[len - 1] = ','; + } + } + + /* common H2C & C2H */ + if ((v & XDMA_STAT_DESC_ERR_MASK)) { + len += sprintf(buf + len, "DESC_ERR:"); + if ((v & XDMA_STAT_DESC_UNSUPP_REQ)) + len += sprintf(buf + len, "UNSUPP_REQ "); + if ((v & XDMA_STAT_DESC_COMPL_ABORT)) + len += sprintf(buf + len, "COMPL_ABORT "); + if ((v & XDMA_STAT_DESC_PARITY_ERR)) + len += sprintf(buf + len, "PARITY "); + if ((v & XDMA_STAT_DESC_HEADER_EP)) + len += sprintf(buf + len, "HEADER_EP "); + if ((v & XDMA_STAT_DESC_UNEXP_COMPL)) + len += sprintf(buf + len, "UNEXP_COMPL "); + buf[len - 1] = ','; + } + + buf[len - 1] = '\0'; + pr_info("%s\n", buffer); +} + +static u32 engine_status_read(struct xdma_engine *engine, bool clear, bool dump) +{ + u32 value; + + BUG_ON(!engine); + + if (dump) + engine_reg_dump(engine); + + /* read status register */ + if (clear) + value = engine->status = + read_register(&engine->regs->status_rc); + else + value = engine->status = read_register(&engine->regs->status); + + if (dump) + engine_status_dump(engine); + + return value; +} + +/** + * xdma_engine_stop() - stop an SG DMA engine + * + */ +static void xdma_engine_stop(struct xdma_engine *engine) +{ + u32 w; + + BUG_ON(!engine); + dbg_tfr("xdma_engine_stop(engine=%p)\n", engine); + + w = 0; + w |= (u32)XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + w |= (u32)XDMA_CTRL_IE_MAGIC_STOPPED; + w |= (u32)XDMA_CTRL_IE_READ_ERROR; + w |= (u32)XDMA_CTRL_IE_DESC_ERROR; + + if (poll_mode) { + w |= (u32) XDMA_CTRL_POLL_MODE_WB; + } else { + w |= (u32)XDMA_CTRL_IE_DESC_STOPPED; + w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED; + + /* Disable IDLE STOPPED for MM */ + if ((engine->streaming && (engine->dir == DMA_FROM_DEVICE)) || + (engine->xdma_perf)) + w |= (u32)XDMA_CTRL_IE_IDLE_STOPPED; + } + + dbg_tfr("Stopping SG DMA %s engine; writing 0x%08x to 0x%p.\n", + engine->name, w, (u32 *)&engine->regs->control); + write_register(w, &engine->regs->control, + (unsigned long)(&engine->regs->control) - + (unsigned long)(&engine->regs)); + /* dummy read of status register to flush all previous writes */ + dbg_tfr("xdma_engine_stop(%s) done\n", engine->name); +} + +static void engine_start_mode_config(struct xdma_engine *engine) +{ + u32 w; + + BUG_ON(!engine); + + /* If a perf test is running, enable the engine interrupts */ + if (engine->xdma_perf) { + w = XDMA_CTRL_IE_DESC_STOPPED; + w |= XDMA_CTRL_IE_DESC_COMPLETED; + w |= XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + w |= XDMA_CTRL_IE_MAGIC_STOPPED; + w |= XDMA_CTRL_IE_IDLE_STOPPED; + w |= XDMA_CTRL_IE_READ_ERROR; + w |= XDMA_CTRL_IE_DESC_ERROR; + + write_register(w, &engine->regs->interrupt_enable_mask, + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); + } + + /* write control register of SG DMA engine */ + w = (u32)XDMA_CTRL_RUN_STOP; + w |= (u32)XDMA_CTRL_IE_READ_ERROR; + w |= (u32)XDMA_CTRL_IE_DESC_ERROR; + w |= (u32)XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + w |= (u32)XDMA_CTRL_IE_MAGIC_STOPPED; + + if (poll_mode) { + w |= (u32)XDMA_CTRL_POLL_MODE_WB; + } else { + w |= (u32)XDMA_CTRL_IE_DESC_STOPPED; + w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED; + + if ((engine->streaming && (engine->dir == DMA_FROM_DEVICE)) || + (engine->xdma_perf)) + w |= (u32)XDMA_CTRL_IE_IDLE_STOPPED; + + /* set non-incremental addressing mode */ + if (engine->non_incr_addr) + w |= (u32)XDMA_CTRL_NON_INCR_ADDR; + } + + dbg_tfr("iowrite32(0x%08x to 0x%p) (control)\n", w, + (void *)&engine->regs->control); + /* start the engine */ + write_register(w, &engine->regs->control, + (unsigned long)(&engine->regs->control) - + (unsigned long)(&engine->regs)); + + /* dummy read of status register to flush all previous writes */ + w = read_register(&engine->regs->status); + dbg_tfr("ioread32(0x%p) = 0x%08x (dummy read flushes writes).\n", + &engine->regs->status, w); +} + +/** + * engine_start() - start an idle engine with its first transfer on queue + * + * The engine will run and process all transfers that are queued using + * transfer_queue() and thus have their descriptor lists chained. + * + * During the run, new transfers will be processed if transfer_queue() has + * chained the descriptors before the hardware fetches the last descriptor. + * A transfer that was chained too late will invoke a new run of the engine + * initiated from the engine_service() routine. + * + * The engine must be idle and at least one transfer must be queued. + * This function does not take locks; the engine spinlock must already be + * taken. + * + */ +static struct xdma_transfer *engine_start(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer; + u32 w; + int extra_adj = 0; + + /* engine must be idle */ + BUG_ON(engine->running); + /* engine transfer queue must not be empty */ + BUG_ON(list_empty(&engine->transfer_list)); + /* inspect first transfer queued on the engine */ + transfer = list_entry(engine->transfer_list.next, struct xdma_transfer, + entry); + BUG_ON(!transfer); + + /* engine is no longer shutdown */ + engine->shutdown = ENGINE_SHUTDOWN_NONE; + + dbg_tfr("engine_start(%s): transfer=0x%p.\n", engine->name, transfer); + + /* initialize number of descriptors of dequeued transfers */ + engine->desc_dequeued = 0; + + /* write lower 32-bit of bus address of transfer first descriptor */ + w = cpu_to_le32(PCI_DMA_L(transfer->desc_bus)); + dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_lo)\n", w, + (void *)&engine->sgdma_regs->first_desc_lo); + write_register(w, &engine->sgdma_regs->first_desc_lo, + (unsigned long)(&engine->sgdma_regs->first_desc_lo) - + (unsigned long)(&engine->sgdma_regs)); + /* write upper 32-bit of bus address of transfer first descriptor */ + w = cpu_to_le32(PCI_DMA_H(transfer->desc_bus)); + dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_hi)\n", w, + (void *)&engine->sgdma_regs->first_desc_hi); + write_register(w, &engine->sgdma_regs->first_desc_hi, + (unsigned long)(&engine->sgdma_regs->first_desc_hi) - + (unsigned long)(&engine->sgdma_regs)); + + if (transfer->desc_adjacent > 0) { + extra_adj = transfer->desc_adjacent - 1; + if (extra_adj > MAX_EXTRA_ADJ) + extra_adj = MAX_EXTRA_ADJ; + } + dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_adjacent)\n", + extra_adj, (void *)&engine->sgdma_regs->first_desc_adjacent); + write_register(extra_adj, &engine->sgdma_regs->first_desc_adjacent, + (unsigned long)(&engine->sgdma_regs->first_desc_adjacent) - + (unsigned long)(&engine->sgdma_regs)); + + dbg_tfr("ioread32(0x%p) (dummy read flushes writes).\n", + &engine->regs->status); + + #if LINUX_VERSION_CODE < KERNEL_VERSION(5,3,0) + mmiowb(); + #endif + + engine_start_mode_config(engine); + + engine_status_read(engine, 0, 0); + + dbg_tfr("%s engine 0x%p now running\n", engine->name, engine); + /* remember the engine is running */ + engine->running = 1; + return transfer; +} + +/** + * engine_service() - service an SG DMA engine + * + * must be called with engine->lock already acquired + * + * @engine pointer to struct xdma_engine + * + */ +static void engine_service_shutdown(struct xdma_engine *engine) +{ + /* if the engine stopped with RUN still asserted, de-assert RUN now */ + dbg_tfr("engine just went idle, resetting RUN_STOP.\n"); + xdma_engine_stop(engine); + engine->running = 0; + + /* awake task on engine's shutdown wait queue */ + wake_up_interruptible(&engine->shutdown_wq); +} + +struct xdma_transfer *engine_transfer_completion(struct xdma_engine *engine, + struct xdma_transfer *transfer) +{ + BUG_ON(!engine); + + if (unlikely(!transfer)) { + pr_info("%s: xfer empty.\n", engine->name); + return NULL; + } + + /* synchronous I/O? */ + /* awake task on transfer's wait queue */ + wake_up_interruptible(&transfer->wq); + + return transfer; +} + +struct xdma_transfer *engine_service_transfer_list(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 *pdesc_completed) +{ + BUG_ON(!engine); + BUG_ON(!pdesc_completed); + + if (unlikely(!transfer)) { + pr_info("%s xfer empty, pdesc completed %u.\n", + engine->name, *pdesc_completed); + return NULL; + } + + /* + * iterate over all the transfers completed by the engine, + * except for the last (i.e. use > instead of >=). + */ + while (transfer && (!transfer->cyclic) && + (*pdesc_completed > transfer->desc_num)) { + /* remove this transfer from pdesc_completed */ + *pdesc_completed -= transfer->desc_num; + dbg_tfr("%s engine completed non-cyclic xfer 0x%p (%d desc)\n", + engine->name, transfer, transfer->desc_num); + /* remove completed transfer from list */ + list_del(engine->transfer_list.next); + /* add to dequeued number of descriptors during this run */ + engine->desc_dequeued += transfer->desc_num; + /* mark transfer as succesfully completed */ + transfer->state = TRANSFER_STATE_COMPLETED; + + /* Complete transfer - sets transfer to NULL if an async + * transfer has completed */ + transfer = engine_transfer_completion(engine, transfer); + + /* if exists, get the next transfer on the list */ + if (!list_empty(&engine->transfer_list)) { + transfer = list_entry(engine->transfer_list.next, + struct xdma_transfer, entry); + dbg_tfr("Non-completed transfer %p\n", transfer); + } else { + /* no further transfers? */ + transfer = NULL; + } + } + + return transfer; +} + +static void engine_err_handle(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 desc_completed) +{ + u32 value; + + /* + * The BUSY bit is expected to be clear now but older HW has a race + * condition which could cause it to be still set. If it's set, re-read + * and check again. If it's still set, log the issue. + */ + if (engine->status & XDMA_STAT_BUSY) { + value = read_register(&engine->regs->status); + if ((value & XDMA_STAT_BUSY) && printk_ratelimit()) + pr_info("%s has errors but is still BUSY\n", + engine->name); + } + + if (printk_ratelimit()) { + pr_info("%s, s 0x%x, aborted xfer 0x%p, cmpl %d/%d\n", + engine->name, engine->status, transfer, desc_completed, + transfer->desc_num); + } + + /* mark transfer as failed */ + transfer->state = TRANSFER_STATE_FAILED; + xdma_engine_stop(engine); +} + +struct xdma_transfer *engine_service_final_transfer(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 *pdesc_completed) +{ + BUG_ON(!engine); + BUG_ON(!pdesc_completed); + + /* inspect the current transfer */ + if (unlikely(!transfer)) { + pr_info("%s xfer empty, pdesc completed %u.\n", + engine->name, *pdesc_completed); + return NULL; + } else { + if (((engine->dir == DMA_FROM_DEVICE) && + (engine->status & XDMA_STAT_C2H_ERR_MASK)) || + ((engine->dir == DMA_TO_DEVICE) && + (engine->status & XDMA_STAT_H2C_ERR_MASK))) { + pr_info("engine %s, status error 0x%x.\n", + engine->name, engine->status); + engine_status_dump(engine); + engine_err_handle(engine, transfer, *pdesc_completed); + goto transfer_del; + } + + if (engine->status & XDMA_STAT_BUSY) + pr_debug("engine %s is unexpectedly busy - ignoring\n", + engine->name); + + /* the engine stopped on current transfer? */ + if (*pdesc_completed < transfer->desc_num) { + transfer->state = TRANSFER_STATE_FAILED; + pr_info("%s, xfer 0x%p, stopped half-way, %d/%d.\n", + engine->name, transfer, *pdesc_completed, + transfer->desc_num); + } else { + dbg_tfr("engine %s completed transfer\n", engine->name); + dbg_tfr("Completed transfer ID = 0x%p\n", transfer); + dbg_tfr("*pdesc_completed=%d, transfer->desc_num=%d", + *pdesc_completed, transfer->desc_num); + + if (!transfer->cyclic) { + /* + * if the engine stopped on this transfer, + * it should be the last + */ + WARN_ON(*pdesc_completed > transfer->desc_num); + } + /* mark transfer as succesfully completed */ + transfer->state = TRANSFER_STATE_COMPLETED; + } + +transfer_del: + /* remove completed transfer from list */ + list_del(engine->transfer_list.next); + /* add to dequeued number of descriptors during this run */ + engine->desc_dequeued += transfer->desc_num; + + /* + * Complete transfer - sets transfer to NULL if an asynchronous + * transfer has completed + */ + transfer = engine_transfer_completion(engine, transfer); + } + + return transfer; +} + +static void engine_service_perf(struct xdma_engine *engine, u32 desc_completed) +{ + BUG_ON(!engine); + + /* performance measurement is running? */ + if (engine->xdma_perf) { + /* a descriptor was completed? */ + if (engine->status & XDMA_STAT_DESC_COMPLETED) { + engine->xdma_perf->iterations = desc_completed; + dbg_perf("transfer->xdma_perf->iterations=%d\n", + engine->xdma_perf->iterations); + } + + /* a descriptor stopped the engine? */ + if (engine->status & XDMA_STAT_DESC_STOPPED) { + engine->xdma_perf->stopped = 1; + /* + * wake any XDMA_PERF_IOCTL_STOP waiting for + * the performance run to finish + */ + wake_up_interruptible(&engine->xdma_perf_wq); + dbg_perf("transfer->xdma_perf stopped\n"); + } + } +} + +static void engine_transfer_dequeue(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer; + + BUG_ON(!engine); + + /* pick first transfer on the queue (was submitted to the engine) */ + transfer = list_entry(engine->transfer_list.next, struct xdma_transfer, + entry); + if (!transfer || transfer != &engine->cyclic_req->xfer) { + pr_info("%s, xfer 0x%p != 0x%p.\n", + engine->name, transfer, &engine->cyclic_req->xfer); + return; + } + dbg_tfr("%s engine completed cyclic transfer 0x%p (%d desc).\n", + engine->name, transfer, transfer->desc_num); + /* remove completed transfer from list */ + list_del(engine->transfer_list.next); +} + +static int engine_ring_process(struct xdma_engine *engine) +{ + struct xdma_result *result; + int start; + int eop_count = 0; + + BUG_ON(!engine); + result = engine->cyclic_result; + BUG_ON(!result); + + /* where we start receiving in the ring buffer */ + start = engine->rx_tail; + + /* iterate through all newly received RX result descriptors */ + dbg_tfr("%s, result %d, 0x%x, len 0x%x.\n", + engine->name, engine->rx_tail, result[engine->rx_tail].status, + result[engine->rx_tail].length); + while (result[engine->rx_tail].status && !engine->rx_overrun) { + /* EOP bit set in result? */ + if (result[engine->rx_tail].status & RX_STATUS_EOP){ + eop_count++; + } + + /* increment tail pointer */ + engine->rx_tail = (engine->rx_tail + 1) % CYCLIC_RX_PAGES_MAX; + + dbg_tfr("%s, head %d, tail %d, 0x%x, len 0x%x.\n", + engine->name, engine->rx_head, engine->rx_tail, + result[engine->rx_tail].status, + result[engine->rx_tail].length); + + /* overrun? */ + if (engine->rx_tail == engine->rx_head) { + dbg_tfr("%s: overrun\n", engine->name); + /* flag to user space that overrun has occurred */ + engine->rx_overrun = 1; + } + } + + return eop_count; +} + +static int engine_service_cyclic_polled(struct xdma_engine *engine) +{ + int eop_count = 0; + int rc = 0; + struct xdma_poll_wb *writeback_data; + u32 sched_limit = 0; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + writeback_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + + while (eop_count == 0) { + if (sched_limit != 0) { + if ((sched_limit % NUM_POLLS_PER_SCHED) == 0) + schedule(); + } + sched_limit++; + + /* Monitor descriptor writeback address for errors */ + if ((writeback_data->completed_desc_count) & WB_ERR_MASK) { + rc = -1; + break; + } + + eop_count = engine_ring_process(engine); + } + + if (eop_count == 0) { + engine_status_read(engine, 1, 0); + if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) + engine_transfer_dequeue(engine); + + engine_service_shutdown(engine); + } + } + + return rc; +} + +static int engine_service_cyclic_interrupt(struct xdma_engine *engine) +{ + int eop_count = 0; + struct xdma_transfer *xfer; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + engine_status_read(engine, 1, 0); + + eop_count = engine_ring_process(engine); + /* + * wake any reader on EOP, as one or more packets are now in + * the RX buffer + */ + xfer = &engine->cyclic_req->xfer; + if(enable_credit_mp){ + if (eop_count > 0) { + //engine->eop_found = 1; + } + wake_up_interruptible(&xfer->wq); + }else{ + if (eop_count > 0) { + /* awake task on transfer's wait queue */ + dbg_tfr("wake_up_interruptible() due to %d EOP's\n", eop_count); + engine->eop_found = 1; + wake_up_interruptible(&xfer->wq); + } + } + + /* engine was running but is no longer busy? */ + if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) + engine_transfer_dequeue(engine); + + engine_service_shutdown(engine); + } + + return 0; +} + +/* must be called with engine->lock already acquired */ +static int engine_service_cyclic(struct xdma_engine *engine) +{ + int rc = 0; + + dbg_tfr("engine_service_cyclic()"); + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + if (poll_mode) + rc = engine_service_cyclic_polled(engine); + else + rc = engine_service_cyclic_interrupt(engine); + + return rc; +} + + +static void engine_service_resume(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer_started; + + BUG_ON(!engine); + + /* engine stopped? */ + if (!engine->running) { + /* in the case of shutdown, let it finish what's in the Q */ + if (!list_empty(&engine->transfer_list)) { + /* (re)start engine */ + transfer_started = engine_start(engine); + pr_info("re-started %s engine with pending xfer 0x%p\n", + engine->name, transfer_started); + /* engine was requested to be shutdown? */ + } else if (engine->shutdown & ENGINE_SHUTDOWN_REQUEST) { + engine->shutdown |= ENGINE_SHUTDOWN_IDLE; + /* awake task on engine's shutdown wait queue */ + wake_up_interruptible(&engine->shutdown_wq); + } else { + dbg_tfr("no pending transfers, %s engine stays idle.\n", + engine->name); + } + } else { + /* engine is still running? */ + if (list_empty(&engine->transfer_list)) { + pr_warn("no queued transfers but %s engine running!\n", + engine->name); + WARN_ON(1); + } + } +} + +/** + * engine_service() - service an SG DMA engine + * + * must be called with engine->lock already acquired + * + * @engine pointer to struct xdma_engine + * + */ +static int engine_service(struct xdma_engine *engine, int desc_writeback) +{ + struct xdma_transfer *transfer = NULL; + u32 desc_count = desc_writeback & WB_COUNT_MASK; + u32 err_flag = desc_writeback & WB_ERR_MASK; + int rv = 0; + struct xdma_poll_wb *wb_data; + + BUG_ON(!engine); + + /* If polling detected an error, signal to the caller */ + if (err_flag) + rv = -1; + + /* Service the engine */ + if (!engine->running) { + dbg_tfr("Engine was not running!!! Clearing status\n"); + engine_status_read(engine, 1, 0); + return 0; + } + + /* + * If called by the ISR or polling detected an error, read and clear + * engine status. For polled mode descriptor completion, this read is + * unnecessary and is skipped to reduce latency + */ + if ((desc_count == 0) || (err_flag != 0)) + engine_status_read(engine, 1, 0); + + /* + * engine was running but is no longer busy, or writeback occurred, + * shut down + */ + if ((engine->running && !(engine->status & XDMA_STAT_BUSY)) || + (desc_count != 0)) + engine_service_shutdown(engine); + + /* + * If called from the ISR, or if an error occurred, the descriptor + * count will be zero. In this scenario, read the descriptor count + * from HW. In polled mode descriptor completion, this read is + * unnecessary and is skipped to reduce latency + */ + if (!desc_count) + desc_count = read_register(&engine->regs->completed_desc_count); + dbg_tfr("desc_count = %d\n", desc_count); + + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) { + /* pick first transfer on queue (was submitted to the engine) */ + transfer = list_entry(engine->transfer_list.next, + struct xdma_transfer, entry); + + dbg_tfr("head of queue transfer 0x%p has %d descriptors\n", + transfer, (int)transfer->desc_num); + + dbg_tfr("Engine completed %d desc, %d not yet dequeued\n", + (int)desc_count, + (int)desc_count - engine->desc_dequeued); + + engine_service_perf(engine, desc_count); + } + + /* account for already dequeued transfers during this engine run */ + desc_count -= engine->desc_dequeued; + + /* Process all but the last transfer */ + transfer = engine_service_transfer_list(engine, transfer, &desc_count); + + /* + * Process final transfer - includes checks of number of descriptors to + * detect faulty completion + */ + transfer = engine_service_final_transfer(engine, transfer, &desc_count); + + /* Before starting engine again, clear the writeback data */ + if (poll_mode) { + wb_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + wb_data->completed_desc_count = 0; + } + + /* Restart the engine following the servicing */ + engine_service_resume(engine); + + return 0; +} + +/* engine_service_work */ +static void engine_service_work(struct work_struct *work) +{ + struct xdma_engine *engine; + unsigned long flags; + + engine = container_of(work, struct xdma_engine, work); + BUG_ON(engine->magic != MAGIC_ENGINE); + + /* lock the engine */ + spin_lock_irqsave(&engine->lock, flags); + + dbg_tfr("engine_service() for %s engine %p\n", + engine->name, engine); + if (engine->cyclic_req) + engine_service_cyclic(engine); + else + engine_service(engine, 0); + + /* re-enable interrupts for this engine */ + if (engine->xdev->msix_enabled){ + write_register(engine->interrupt_enable_mask_value, + &engine->regs->interrupt_enable_mask_w1s, + (unsigned long)(&engine->regs->interrupt_enable_mask_w1s) - + (unsigned long)(&engine->regs)); + } else + channel_interrupts_enable(engine->xdev, engine->irq_bitmask); + + /* unlock the engine */ + spin_unlock_irqrestore(&engine->lock, flags); +} + +static u32 engine_service_wb_monitor(struct xdma_engine *engine, + u32 expected_wb) +{ + struct xdma_poll_wb *wb_data; + u32 desc_wb = 0; + u32 sched_limit = 0; + unsigned long timeout; + + BUG_ON(!engine); + wb_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + + /* + * Poll the writeback location for the expected number of + * descriptors / error events This loop is skipped for cyclic mode, + * where the expected_desc_count passed in is zero, since it cannot be + * determined before the function is called + */ + + timeout = jiffies + (POLL_TIMEOUT_SECONDS * HZ); + while (expected_wb != 0) { + desc_wb = wb_data->completed_desc_count; + + if (desc_wb & WB_ERR_MASK) + break; + else if (desc_wb == expected_wb) + break; + + /* RTO - prevent system from hanging in polled mode */ + if (time_after(jiffies, timeout)) { + dbg_tfr("Polling timeout occurred"); + dbg_tfr("desc_wb = 0x%08x, expected 0x%08x\n", desc_wb, + expected_wb); + if ((desc_wb & WB_COUNT_MASK) > expected_wb) + desc_wb = expected_wb | WB_ERR_MASK; + + break; + } + + /* + * Define NUM_POLLS_PER_SCHED to limit how much time is spent + * in the scheduler + */ + + if (sched_limit != 0) { + if ((sched_limit % NUM_POLLS_PER_SCHED) == 0) + schedule(); + } + sched_limit++; + } + + return desc_wb; +} + +static int engine_service_poll(struct xdma_engine *engine, + u32 expected_desc_count) +{ + struct xdma_poll_wb *writeback_data; + u32 desc_wb = 0; + unsigned long flags; + int rv = 0; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + writeback_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + + if ((expected_desc_count & WB_COUNT_MASK) != expected_desc_count) { + dbg_tfr("Queued descriptor count is larger than supported\n"); + return -1; + } + + /* + * Poll the writeback location for the expected number of + * descriptors / error events This loop is skipped for cyclic mode, + * where the expected_desc_count passed in is zero, since it cannot be + * determined before the function is called + */ + + desc_wb = engine_service_wb_monitor(engine, expected_desc_count); + + spin_lock_irqsave(&engine->lock, flags); + dbg_tfr("%s service.\n", engine->name); + if (engine->cyclic_req) { + rv = engine_service_cyclic(engine); + } else { + rv = engine_service(engine, desc_wb); + } + spin_unlock_irqrestore(&engine->lock, flags); + + return rv; +} + +static irqreturn_t user_irq_service(int irq, struct xdma_user_irq *user_irq) +{ + unsigned long flags; + + BUG_ON(!user_irq); + + if (user_irq->handler) + return user_irq->handler(user_irq->user_idx, user_irq->dev); + + spin_lock_irqsave(&(user_irq->events_lock), flags); + if (!user_irq->events_irq) { + user_irq->events_irq = 1; + wake_up_interruptible(&(user_irq->events_wq)); + } + spin_unlock_irqrestore(&(user_irq->events_lock), flags); + + return IRQ_HANDLED; +} + +/* + * xdma_isr() - Interrupt handler + * + * @dev_id pointer to xdma_dev + */ +static irqreturn_t xdma_isr(int irq, void *dev_id) +{ + u32 ch_irq; + u32 user_irq; + u32 mask; + struct xdma_dev *xdev; + struct interrupt_regs *irq_regs; + + dbg_irq("(irq=%d, dev 0x%p) <<<< ISR.\n", irq, dev_id); + BUG_ON(!dev_id); + xdev = (struct xdma_dev *)dev_id; + + if (!xdev) { + WARN_ON(!xdev); + dbg_irq("xdma_isr(irq=%d) xdev=%p ??\n", irq, xdev); + return IRQ_NONE; + } + + irq_regs = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + + /* read channel interrupt requests */ + ch_irq = read_register(&irq_regs->channel_int_request); + dbg_irq("ch_irq = 0x%08x\n", ch_irq); + + /* + * disable all interrupts that fired; these are re-enabled individually + * after the causing module has been fully serviced. + */ + if (ch_irq) + channel_interrupts_disable(xdev, ch_irq); + + /* read user interrupts - this read also flushes the above write */ + user_irq = read_register(&irq_regs->user_int_request); + dbg_irq("user_irq = 0x%08x\n", user_irq); + + if (user_irq) { + int user = 0; + u32 mask = 1; + int max = xdev->h2c_channel_max; + + for (; user < max && user_irq; user++, mask <<= 1) { + if (user_irq & mask) { + user_irq &= ~mask; + user_irq_service(irq, &xdev->user_irq[user]); + } + } + } + + mask = ch_irq & xdev->mask_irq_h2c; + if (mask) { + int channel = 0; + int max = xdev->h2c_channel_max; + + /* iterate over H2C (PCIe read) */ + for (channel = 0; channel < max && mask; channel++) { + struct xdma_engine *engine = &xdev->engine_h2c[channel]; + + /* engine present and its interrupt fired? */ + if((engine->irq_bitmask & mask) && + (engine->magic == MAGIC_ENGINE)) { + mask &= ~engine->irq_bitmask; + dbg_tfr("schedule_work, %s.\n", engine->name); + schedule_work(&engine->work); + } + } + } + + mask = ch_irq & xdev->mask_irq_c2h; + if (mask) { + int channel = 0; + int max = xdev->c2h_channel_max; + + /* iterate over C2H (PCIe write) */ + for (channel = 0; channel < max && mask; channel++) { + struct xdma_engine *engine = &xdev->engine_c2h[channel]; + + /* engine present and its interrupt fired? */ + if((engine->irq_bitmask & mask) && + (engine->magic == MAGIC_ENGINE)) { + mask &= ~engine->irq_bitmask; + dbg_tfr("schedule_work, %s.\n", engine->name); + schedule_work(&engine->work); + } + } + } + + xdev->irq_count++; + return IRQ_HANDLED; +} + +/* + * xdma_user_irq() - Interrupt handler for user interrupts in MSI-X mode + * + * @dev_id pointer to xdma_dev + */ +static irqreturn_t xdma_user_irq(int irq, void *dev_id) +{ + struct xdma_user_irq *user_irq; + + dbg_irq("(irq=%d) <<<< INTERRUPT SERVICE ROUTINE\n", irq); + + BUG_ON(!dev_id); + user_irq = (struct xdma_user_irq *)dev_id; + + return user_irq_service(irq, user_irq); +} + +/* + * xdma_channel_irq() - Interrupt handler for channel interrupts in MSI-X mode + * + * @dev_id pointer to xdma_dev + */ +static irqreturn_t xdma_channel_irq(int irq, void *dev_id) +{ + struct xdma_dev *xdev; + struct xdma_engine *engine; + struct interrupt_regs *irq_regs; + + dbg_irq("(irq=%d) <<<< INTERRUPT service ROUTINE\n", irq); + BUG_ON(!dev_id); + + engine = (struct xdma_engine *)dev_id; + xdev = engine->xdev; + + if (!xdev) { + WARN_ON(!xdev); + dbg_irq("xdma_channel_irq(irq=%d) xdev=%p ??\n", irq, xdev); + return IRQ_NONE; + } + + irq_regs = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + + /* Disable the interrupt for this engine */ + write_register(engine->interrupt_enable_mask_value, + &engine->regs->interrupt_enable_mask_w1c, + (unsigned long) + (&engine->regs->interrupt_enable_mask_w1c) - + (unsigned long)(&engine->regs)); + /* Dummy read to flush the above write */ + read_register(&irq_regs->channel_int_pending); + /* Schedule the bottom half */ + schedule_work(&engine->work); + + /* + * RTO - need to protect access here if multiple MSI-X are used for + * user interrupts + */ + xdev->irq_count++; + return IRQ_HANDLED; +} + +/* + * Unmap the BAR regions that had been mapped earlier using map_bars() + */ +static void unmap_bars(struct xdma_dev *xdev, struct pci_dev *dev) +{ + int i; + + for (i = 0; i < XDMA_BAR_NUM; i++) { + /* is this BAR mapped? */ + if (xdev->bar[i]) { + /* unmap BAR */ + pci_iounmap(dev, xdev->bar[i]); + /* mark as unmapped */ + xdev->bar[i] = NULL; + } + } +} + +static int map_single_bar(struct xdma_dev *xdev, struct pci_dev *dev, int idx) +{ + resource_size_t bar_start; + resource_size_t bar_len; + resource_size_t map_len; + + bar_start = pci_resource_start(dev, idx); + bar_len = pci_resource_len(dev, idx); + map_len = bar_len; + + xdev->bar[idx] = NULL; + + /* do not map BARs with length 0. Note that start MAY be 0! */ + if (!bar_len) { + //pr_info("BAR #%d is not present - skipping\n", idx); + return 0; + } + + /* BAR size exceeds maximum desired mapping? */ + if (bar_len > INT_MAX) { + pr_info("Limit BAR %d mapping from %llu to %d bytes\n", idx, + (u64)bar_len, INT_MAX); + map_len = (resource_size_t)INT_MAX; + } + /* + * map the full device memory or IO region into kernel virtual + * address space + */ + dbg_init("BAR%d: %llu bytes to be mapped.\n", idx, (u64)map_len); + xdev->bar[idx] = pci_iomap(dev, idx, map_len); + + if (!xdev->bar[idx]) { + pr_info("Could not map BAR %d.\n", idx); + return -1; + } + + pr_info("BAR%d at 0x%llx mapped at 0x%p, length=%llu(/%llu)\n", idx, + (u64)bar_start, xdev->bar[idx], (u64)map_len, (u64)bar_len); + + return (int)map_len; +} + +static int is_config_bar(struct xdma_dev *xdev, int idx) +{ + u32 irq_id = 0; + u32 cfg_id = 0; + int flag = 0; + u32 mask = 0xffff0000; /* Compare only XDMA ID's not Version number */ + struct interrupt_regs *irq_regs = + (struct interrupt_regs *) (xdev->bar[idx] + XDMA_OFS_INT_CTRL); + struct config_regs *cfg_regs = + (struct config_regs *)(xdev->bar[idx] + XDMA_OFS_CONFIG); + + irq_id = read_register(&irq_regs->identifier); + cfg_id = read_register(&cfg_regs->identifier); + + if (((irq_id & mask)== IRQ_BLOCK_ID) && + ((cfg_id & mask)== CONFIG_BLOCK_ID)) { + dbg_init("BAR %d is the XDMA config BAR\n", idx); + flag = 1; + } else { + dbg_init("BAR %d is NOT the XDMA config BAR: 0x%x, 0x%x.\n", + idx, irq_id, cfg_id); + flag = 0; + } + + return flag; +} + +static void identify_bars(struct xdma_dev *xdev, int *bar_id_list, int num_bars, + int config_bar_pos) +{ + /* + * The following logic identifies which BARs contain what functionality + * based on the position of the XDMA config BAR and the number of BARs + * detected. The rules are that the user logic and bypass logic BARs + * are optional. When both are present, the XDMA config BAR will be the + * 2nd BAR detected (config_bar_pos = 1), with the user logic being + * detected first and the bypass being detected last. When one is + * omitted, the type of BAR present can be identified by whether the + * XDMA config BAR is detected first or last. When both are omitted, + * only the XDMA config BAR is present. This somewhat convoluted + * approach is used instead of relying on BAR numbers in order to work + * correctly with both 32-bit and 64-bit BARs. + */ + + BUG_ON(!xdev); + BUG_ON(!bar_id_list); + + dbg_init("xdev 0x%p, bars %d, config at %d.\n", + xdev, num_bars, config_bar_pos); + + switch (num_bars) { + case 1: + /* Only one BAR present - no extra work necessary */ + break; + + case 2: + if (config_bar_pos == 0) { + xdev->bypass_bar_idx = bar_id_list[1]; + } else if (config_bar_pos == 1) { + xdev->user_bar_idx = bar_id_list[0]; + } else { + pr_info("2, XDMA config BAR unexpected %d.\n", + config_bar_pos); + } + break; + + case 3: + case 4: + if ((config_bar_pos == 1) || (config_bar_pos == 2)) { + /* user bar at bar #0 */ + xdev->user_bar_idx = bar_id_list[0]; + /* bypass bar at the last bar */ + xdev->bypass_bar_idx = bar_id_list[num_bars - 1]; + } else { + pr_info("3/4, XDMA config BAR unexpected %d.\n", + config_bar_pos); + } + break; + + default: + /* Should not occur - warn user but safe to continue */ + pr_info("Unexpected # BARs (%d), XDMA config BAR only.\n", + num_bars); + break; + + } + pr_info("%d BARs: config %d, user %d, bypass %d.\n", + num_bars, config_bar_pos, xdev->user_bar_idx, + xdev->bypass_bar_idx); +} + +/* map_bars() -- map device regions into kernel virtual address space + * + * Map the device memory regions into kernel virtual address space after + * verifying their sizes respect the minimum sizes needed + */ +static int map_bars(struct xdma_dev *xdev, struct pci_dev *dev) +{ + int rv; + int i; + int bar_id_list[XDMA_BAR_NUM]; + int bar_id_idx = 0; + int config_bar_pos = 0; + + /* iterate through all the BARs */ + for (i = 0; i < XDMA_BAR_NUM; i++) { + int bar_len; + + bar_len = map_single_bar(xdev, dev, i); + if (bar_len == 0) { + continue; + } else if (bar_len < 0) { + rv = -EINVAL; + goto fail; + } + + /* Try to identify BAR as XDMA control BAR */ + if ((bar_len >= XDMA_BAR_SIZE) && (xdev->config_bar_idx < 0)) { + + if (is_config_bar(xdev, i)) { + xdev->config_bar_idx = i; + config_bar_pos = bar_id_idx; + pr_info("config bar %d, pos %d.\n", + xdev->config_bar_idx, config_bar_pos); + } + } + + bar_id_list[bar_id_idx] = i; + bar_id_idx++; + } + + /* The XDMA config BAR must always be present */ + if (xdev->config_bar_idx < 0) { + pr_info("Failed to detect XDMA config BAR\n"); + rv = -EINVAL; + goto fail; + } + +#ifdef __LIBXDMA_CONFIG_BAR_ONLY__ + /* unmapped all other bars, except XDMA config. bar */ + for (i = 0; i < XDMA_BAR_NUM; i++) { + if (i == xdev->config_bar_idx) + continue; + + /* is this BAR mapped? */ + if (xdev->bar[i]) { + /* unmap BAR */ + pci_iounmap(dev, xdev->bar[i]); + /* mark as unmapped */ + xdev->bar[i] = NULL; + pr_info("unmap non-config bar %d.\n", i); + } + } +#else + identify_bars(xdev, bar_id_list, bar_id_idx, config_bar_pos); +#endif + + /* successfully mapped all required BAR regions */ + return 0; + +fail: + /* unwind; unmap any BARs that we did map */ + unmap_bars(xdev, dev); + return rv; +} + +/* + * MSI-X interrupt: + * <h2c+c2h channel_max> vectors, followed by <user_max> vectors + */ + +/* + * RTO - code to detect if MSI/MSI-X capability exists is derived + * from linux/pci/msi.c - pci_msi_check_device + */ + +#ifndef arch_msi_check_device +int arch_msi_check_device(struct pci_dev *dev, int nvec, int type) +{ + return 0; +} +#endif + +/* type = PCI_CAP_ID_MSI or PCI_CAP_ID_MSIX */ +static int msi_msix_capable(struct pci_dev *dev, int type) +{ + struct pci_bus *bus; + int ret; + + if (!dev || dev->no_msi) + return 0; + + for (bus = dev->bus; bus; bus = bus->parent) + if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) + return 0; + + ret = arch_msi_check_device(dev, 1, type); + if (ret) + return 0; + + if (!pci_find_capability(dev, type)) + return 0; + + return 1; +} + +static void disable_msi_msix(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + if (xdev->msix_enabled) { + pci_disable_msix(pdev); + xdev->msix_enabled = 0; + } else if (xdev->msi_enabled) { + pci_disable_msi(pdev); + xdev->msi_enabled = 0; + } +} + +static int enable_msi_msix(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + int rv = 0; + + BUG_ON(!xdev); + BUG_ON(!pdev); + + if (!interrupt_mode && msi_msix_capable(pdev, PCI_CAP_ID_MSIX)) { + int req_nvec = xdev->c2h_channel_max + xdev->h2c_channel_max + + xdev->user_max; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + dbg_init("Enabling MSI-X\n"); + rv = pci_alloc_irq_vectors(pdev, req_nvec, req_nvec, + PCI_IRQ_MSIX); +#else + int i; + + dbg_init("Enabling MSI-X\n"); + for (i = 0; i < req_nvec; i++) + xdev->entry[i].entry = i; + + rv = pci_enable_msix(pdev, xdev->entry, req_nvec); +#endif + if (rv < 0) + dbg_init("Couldn't enable MSI-X mode: %d\n", rv); + + xdev->msix_enabled = 1; + + } else if (interrupt_mode == 1 && + msi_msix_capable(pdev, PCI_CAP_ID_MSI)) { + /* enable message signalled interrupts */ + dbg_init("pci_enable_msi()\n"); + rv = pci_enable_msi(pdev); + if (rv < 0) + dbg_init("Couldn't enable MSI mode: %d\n", rv); + xdev->msi_enabled = 1; + + } else { + dbg_init("MSI/MSI-X not detected - using legacy interrupts\n"); + } + + return rv; +} + +static void pci_check_intr_pend(struct pci_dev *pdev) +{ + u16 v; + + pci_read_config_word(pdev, PCI_STATUS, &v); + if (v & PCI_STATUS_INTERRUPT) { + pr_info("%s PCI STATUS Interrupt pending 0x%x.\n", + dev_name(&pdev->dev), v); + pci_write_config_word(pdev, PCI_STATUS, PCI_STATUS_INTERRUPT); + } +} + +static void pci_keep_intx_enabled(struct pci_dev *pdev) +{ + /* workaround to a h/w bug: + * when msix/msi become unavaile, default to legacy. + * However the legacy enable was not checked. + * If the legacy was disabled, no ack then everything stuck + */ + u16 pcmd, pcmd_new; + + pci_read_config_word(pdev, PCI_COMMAND, &pcmd); + pcmd_new = pcmd & ~PCI_COMMAND_INTX_DISABLE; + if (pcmd_new != pcmd) { + pr_info("%s: clear INTX_DISABLE, 0x%x -> 0x%x.\n", + dev_name(&pdev->dev), pcmd, pcmd_new); + pci_write_config_word(pdev, PCI_COMMAND, pcmd_new); + } +} + +static void prog_irq_msix_user(struct xdma_dev *xdev, bool clear) +{ + /* user */ + struct interrupt_regs *int_regs = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + u32 i = xdev->c2h_channel_max + xdev->h2c_channel_max; + u32 max = i + xdev->user_max; + int j; + + for (j = 0; i < max; j++) { + u32 val = 0; + int k; + int shift = 0; + + if (clear) + i += 4; + else + for (k = 0; k < 4 && i < max; i++, k++, shift += 8) + val |= (i & 0x1f) << shift; + + write_register(val, &int_regs->user_msi_vector[j], + XDMA_OFS_INT_CTRL + + ((unsigned long)&int_regs->user_msi_vector[j] - + (unsigned long)int_regs)); + + dbg_init("vector %d, 0x%x.\n", j, val); + } +} + +static void prog_irq_msix_channel(struct xdma_dev *xdev, bool clear) +{ + struct interrupt_regs *int_regs = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + u32 max = xdev->c2h_channel_max + xdev->h2c_channel_max; + u32 i; + int j; + + /* engine */ + for (i = 0, j = 0; i < max; j++) { + u32 val = 0; + int k; + int shift = 0; + + if (clear) + i += 4; + else + for (k = 0; k < 4 && i < max; i++, k++, shift += 8) + val |= (i & 0x1f) << shift; + + write_register(val, &int_regs->channel_msi_vector[j], + XDMA_OFS_INT_CTRL + + ((unsigned long)&int_regs->channel_msi_vector[j] - + (unsigned long)int_regs)); + dbg_init("vector %d, 0x%x.\n", j, val); + } +} + +static void irq_msix_channel_teardown(struct xdma_dev *xdev) +{ + struct xdma_engine *engine; + int j = 0; + int i = 0; + + if (!xdev->msix_enabled) + return; + + prog_irq_msix_channel(xdev, 1); + + engine = xdev->engine_h2c; + for (i = 0; i < xdev->h2c_channel_max; i++, j++, engine++) { + if (!engine->msix_irq_line) + break; + dbg_sg("Release IRQ#%d for engine %p\n", engine->msix_irq_line, + engine); + free_irq(engine->msix_irq_line, engine); + } + + engine = xdev->engine_c2h; + for (i = 0; i < xdev->c2h_channel_max; i++, j++, engine++) { + if (!engine->msix_irq_line) + break; + dbg_sg("Release IRQ#%d for engine %p\n", engine->msix_irq_line, + engine); + free_irq(engine->msix_irq_line, engine); + } +} + +static int irq_msix_channel_setup(struct xdma_dev *xdev) +{ + int i; + int j = xdev->h2c_channel_max; + int rv = 0; + u32 vector; + struct xdma_engine *engine; + + BUG_ON(!xdev); + if (!xdev->msix_enabled) + return 0; + + engine = xdev->engine_h2c; + for (i = 0; i < xdev->h2c_channel_max; i++, engine++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + vector = pci_irq_vector(xdev->pdev, i); +#else + vector = xdev->entry[i].vector; +#endif + rv = request_irq(vector, xdma_channel_irq, 0, xdev->mod_name, + engine); + if (rv) { + pr_info("requesti irq#%d failed %d, engine %s.\n", + vector, rv, engine->name); + return rv; + } + pr_info("engine %s, irq#%d.\n", engine->name, vector); + engine->msix_irq_line = vector; + } + + engine = xdev->engine_c2h; + for (i = 0; i < xdev->c2h_channel_max; i++, j++, engine++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + vector = pci_irq_vector(xdev->pdev, j); +#else + vector = xdev->entry[j].vector; +#endif + rv = request_irq(vector, xdma_channel_irq, 0, xdev->mod_name, + engine); + if (rv) { + pr_info("requesti irq#%d failed %d, engine %s.\n", + vector, rv, engine->name); + return rv; + } + pr_info("engine %s, irq#%d.\n", engine->name, vector); + engine->msix_irq_line = vector; + } + + return 0; +} + +static void irq_msix_user_teardown(struct xdma_dev *xdev) +{ + int i; + int j = xdev->h2c_channel_max + xdev->c2h_channel_max; + + BUG_ON(!xdev); + + if (!xdev->msix_enabled) + return; + + prog_irq_msix_user(xdev, 1); + + for (i = 0; i < xdev->user_max; i++, j++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + u32 vector = pci_irq_vector(xdev->pdev, j); +#else + u32 vector = xdev->entry[j].vector; +#endif + dbg_init("user %d, releasing IRQ#%d\n", i, vector); + free_irq(vector, &xdev->user_irq[i]); + } +} + +static int irq_msix_user_setup(struct xdma_dev *xdev) +{ + int i; + int j = xdev->h2c_channel_max + xdev->c2h_channel_max; + int rv = 0; + + /* vectors set in probe_scan_for_msi() */ + for (i = 0; i < xdev->user_max; i++, j++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + u32 vector = pci_irq_vector(xdev->pdev, j); +#else + u32 vector = xdev->entry[j].vector; +#endif + rv = request_irq(vector, xdma_user_irq, 0, xdev->mod_name, + &xdev->user_irq[i]); + if (rv) { + pr_info("user %d couldn't use IRQ#%d, %d\n", + i, vector, rv); + break; + } + pr_info("%d-USR-%d, IRQ#%d with 0x%p\n", xdev->idx, i, vector, + &xdev->user_irq[i]); + } + + /* If any errors occur, free IRQs that were successfully requested */ + if (rv) { + for (i--, j--; i >= 0; i--, j--) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + u32 vector = pci_irq_vector(xdev->pdev, j); +#else + u32 vector = xdev->entry[j].vector; +#endif + free_irq(vector, &xdev->user_irq[i]); + } + } + + return rv; +} + +static int irq_msi_setup(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + int rv; + + xdev->irq_line = (int)pdev->irq; + rv = request_irq(pdev->irq, xdma_isr, 0, xdev->mod_name, xdev); + if (rv) + dbg_init("Couldn't use IRQ#%d, %d\n", pdev->irq, rv); + else + dbg_init("Using IRQ#%d with 0x%p\n", pdev->irq, xdev); + + return rv; +} + +static int irq_legacy_setup(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + u32 w; + u8 val; + void *reg; + int rv; + + pci_read_config_byte(pdev, PCI_INTERRUPT_PIN, &val); + dbg_init("Legacy Interrupt register value = %d\n", val); + if (val > 1) { + val--; + w = (val<<24) | (val<<16) | (val<<8)| val; + /* Program IRQ Block Channel vactor and IRQ Block User vector + * with Legacy interrupt value */ + reg = xdev->bar[xdev->config_bar_idx] + 0x2080; // IRQ user + write_register(w, reg, 0x2080); + write_register(w, reg+0x4, 0x2084); + write_register(w, reg+0x8, 0x2088); + write_register(w, reg+0xC, 0x208C); + reg = xdev->bar[xdev->config_bar_idx] + 0x20A0; // IRQ Block + write_register(w, reg, 0x20A0); + write_register(w, reg+0x4, 0x20A4); + } + + xdev->irq_line = (int)pdev->irq; + rv = request_irq(pdev->irq, xdma_isr, IRQF_SHARED, xdev->mod_name, + xdev); + if (rv) + dbg_init("Couldn't use IRQ#%d, %d\n", pdev->irq, rv); + else + dbg_init("Using IRQ#%d with 0x%p\n", pdev->irq, xdev); + + return rv; +} + +static void irq_teardown(struct xdma_dev *xdev) +{ + if (xdev->msix_enabled) { + irq_msix_channel_teardown(xdev); + irq_msix_user_teardown(xdev); + } else if (xdev->irq_line != -1) { + dbg_init("Releasing IRQ#%d\n", xdev->irq_line); + free_irq(xdev->irq_line, xdev); + } +} + +static int irq_setup(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + pci_keep_intx_enabled(pdev); + + if (xdev->msix_enabled) { + int rv = irq_msix_channel_setup(xdev); + if (rv) + return rv; + rv = irq_msix_user_setup(xdev); + if (rv) + return rv; + prog_irq_msix_channel(xdev, 0); + prog_irq_msix_user(xdev, 0); + + return 0; + } else if (xdev->msi_enabled) + return irq_msi_setup(xdev, pdev); + + return irq_legacy_setup(xdev, pdev); +} + +#ifdef __LIBXDMA_DEBUG__ +static void dump_desc(struct xdma_desc *desc_virt) +{ + int j; + u32 *p = (u32 *)desc_virt; + static char * const field_name[] = { + "magic|extra_adjacent|control", "bytes", "src_addr_lo", + "src_addr_hi", "dst_addr_lo", "dst_addr_hi", "next_addr", + "next_addr_pad"}; + char *dummy; + + /* remove warning about unused variable when debug printing is off */ + dummy = field_name[0]; + + for (j = 0; j < 8; j += 1) { + pr_info("0x%08lx/0x%02lx: 0x%08x 0x%08x %s\n", + (uintptr_t)p, (uintptr_t)p & 15, (int)*p, + le32_to_cpu(*p), field_name[j]); + p++; + } + pr_info("\n"); +} + +static void transfer_dump(struct xdma_transfer *transfer) +{ + int i; + struct xdma_desc *desc_virt = transfer->desc_virt; + + pr_info("xfer 0x%p, state 0x%x, f 0x%x, dir %d, len %u, last %d.\n", + transfer, transfer->state, transfer->flags, transfer->dir, + transfer->len, transfer->last_in_request); + + pr_info("transfer 0x%p, desc %d, bus 0x%llx, adj %d.\n", + transfer, transfer->desc_num, (u64)transfer->desc_bus, + transfer->desc_adjacent); + for (i = 0; i < transfer->desc_num; i += 1) + dump_desc(desc_virt + i); +} +#endif /* __LIBXDMA_DEBUG__ */ + +/* xdma_desc_alloc() - Allocate cache-coherent array of N descriptors. + * + * Allocates an array of 'number' descriptors in contiguous PCI bus addressable + * memory. Chains the descriptors as a singly-linked list; the descriptor's + * next * pointer specifies the bus address of the next descriptor. + * + * + * @dev Pointer to pci_dev + * @number Number of descriptors to be allocated + * @desc_bus_p Pointer where to store the first descriptor bus address + * + * @return Virtual address of the first descriptor + * + */ +static void transfer_desc_init(struct xdma_transfer *transfer, int count) +{ + struct xdma_desc *desc_virt = transfer->desc_virt; + dma_addr_t desc_bus = transfer->desc_bus; + int i; + int adj = count - 1; + int extra_adj; + u32 temp_control; + + BUG_ON(count > XDMA_TRANSFER_MAX_DESC); + + /* create singly-linked list for SG DMA controller */ + for (i = 0; i < count - 1; i++) { + /* increment bus address to next in array */ + desc_bus += sizeof(struct xdma_desc); + + /* singly-linked list uses bus addresses */ + desc_virt[i].next_lo = cpu_to_le32(PCI_DMA_L(desc_bus)); + desc_virt[i].next_hi = cpu_to_le32(PCI_DMA_H(desc_bus)); + desc_virt[i].bytes = cpu_to_le32(0); + + /* any adjacent descriptors? */ + if (adj > 0) { + extra_adj = adj - 1; + if (extra_adj > MAX_EXTRA_ADJ) + extra_adj = MAX_EXTRA_ADJ; + + adj--; + } else { + extra_adj = 0; + } + + temp_control = DESC_MAGIC | (extra_adj << 8); + + desc_virt[i].control = cpu_to_le32(temp_control); + } + /* { i = number - 1 } */ + /* zero the last descriptor next pointer */ + desc_virt[i].next_lo = cpu_to_le32(0); + desc_virt[i].next_hi = cpu_to_le32(0); + desc_virt[i].bytes = cpu_to_le32(0); + + temp_control = DESC_MAGIC; + + desc_virt[i].control = cpu_to_le32(temp_control); +} + +/* xdma_desc_link() - Link two descriptors + * + * Link the first descriptor to a second descriptor, or terminate the first. + * + * @first first descriptor + * @second second descriptor, or NULL if first descriptor must be set as last. + * @second_bus bus address of second descriptor + */ +static void xdma_desc_link(struct xdma_desc *first, struct xdma_desc *second, + dma_addr_t second_bus) +{ + /* + * remember reserved control in first descriptor, but zero + * extra_adjacent! + */ + /* RTO - what's this about? Shouldn't it be 0x0000c0ffUL? */ + u32 control = le32_to_cpu(first->control) & 0x0000f0ffUL; + /* second descriptor given? */ + if (second) { + /* + * link last descriptor of 1st array to first descriptor of + * 2nd array + */ + first->next_lo = cpu_to_le32(PCI_DMA_L(second_bus)); + first->next_hi = cpu_to_le32(PCI_DMA_H(second_bus)); + WARN_ON(first->next_hi); + /* no second descriptor given */ + } else { + /* first descriptor is the last */ + first->next_lo = 0; + first->next_hi = 0; + } + /* merge magic, extra_adjacent and control field */ + control |= DESC_MAGIC; + + /* write bytes and next_num */ + first->control = cpu_to_le32(control); +} + +/* xdma_desc_adjacent -- Set how many descriptors are adjacent to this one */ +static void xdma_desc_adjacent(struct xdma_desc *desc, int next_adjacent) +{ + int extra_adj = 0; + /* remember reserved and control bits */ + u32 control = le32_to_cpu(desc->control) & 0x0000f0ffUL; + u32 max_adj_4k = 0; + + if (next_adjacent > 0) { + extra_adj = next_adjacent - 1; + if (extra_adj > MAX_EXTRA_ADJ){ + extra_adj = MAX_EXTRA_ADJ; + } + max_adj_4k = (0x1000 - ((le32_to_cpu(desc->next_lo))&0xFFF))/32 - 1; + if (extra_adj>max_adj_4k) { + extra_adj = max_adj_4k; + } + if(extra_adj<0){ + printk("Warning: extra_adj<0, converting it to 0\n"); + extra_adj = 0; + } + } + /* merge adjacent and control field */ + control |= 0xAD4B0000UL | (extra_adj << 8); + /* write control and next_adjacent */ + desc->control = cpu_to_le32(control); +} + +/* xdma_desc_control -- Set complete control field of a descriptor. */ +static void xdma_desc_control_set(struct xdma_desc *first, u32 control_field) +{ + /* remember magic and adjacent number */ + u32 control = le32_to_cpu(first->control) & ~(LS_BYTE_MASK); + + BUG_ON(control_field & ~(LS_BYTE_MASK)); + /* merge adjacent and control field */ + control |= control_field; + /* write control and next_adjacent */ + first->control = cpu_to_le32(control); +} + +/* xdma_desc_clear -- Clear bits in control field of a descriptor. */ +static void xdma_desc_control_clear(struct xdma_desc *first, u32 clear_mask) +{ + /* remember magic and adjacent number */ + u32 control = le32_to_cpu(first->control); + + BUG_ON(clear_mask & ~(LS_BYTE_MASK)); + + /* merge adjacent and control field */ + control &= (~clear_mask); + /* write control and next_adjacent */ + first->control = cpu_to_le32(control); +} + +/* xdma_desc_done - recycle cache-coherent linked list of descriptors. + * + * @dev Pointer to pci_dev + * @number Number of descriptors to be allocated + * @desc_virt Pointer to (i.e. virtual address of) first descriptor in list + * @desc_bus Bus address of first descriptor in list + */ +static inline void xdma_desc_done(struct xdma_desc *desc_virt) +{ + memset(desc_virt, 0, XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc)); +} + +/* xdma_desc() - Fill a descriptor with the transfer details + * + * @desc pointer to descriptor to be filled + * @addr root complex address + * @ep_addr end point address + * @len number of bytes, must be a (non-negative) multiple of 4. + * @dir, dma direction + * is the end point address. If zero, vice versa. + * + * Does not modify the next pointer + */ +static void xdma_desc_set(struct xdma_desc *desc, dma_addr_t rc_bus_addr, + u64 ep_addr, int len, int dir) +{ + /* transfer length */ + desc->bytes = cpu_to_le32(len); + if (dir == DMA_TO_DEVICE) { + /* read from root complex memory (source address) */ + desc->src_addr_lo = cpu_to_le32(PCI_DMA_L(rc_bus_addr)); + desc->src_addr_hi = cpu_to_le32(PCI_DMA_H(rc_bus_addr)); + /* write to end point address (destination address) */ + desc->dst_addr_lo = cpu_to_le32(PCI_DMA_L(ep_addr)); + desc->dst_addr_hi = cpu_to_le32(PCI_DMA_H(ep_addr)); + } else { + /* read from end point address (source address) */ + desc->src_addr_lo = cpu_to_le32(PCI_DMA_L(ep_addr)); + desc->src_addr_hi = cpu_to_le32(PCI_DMA_H(ep_addr)); + /* write to root complex memory (destination address) */ + desc->dst_addr_lo = cpu_to_le32(PCI_DMA_L(rc_bus_addr)); + desc->dst_addr_hi = cpu_to_le32(PCI_DMA_H(rc_bus_addr)); + } +} + +/* + * should hold the engine->lock; + */ +static void transfer_abort(struct xdma_engine *engine, + struct xdma_transfer *transfer) +{ + struct xdma_transfer *head; + + BUG_ON(!engine); + BUG_ON(!transfer); + BUG_ON(transfer->desc_num == 0); + + pr_info("abort transfer 0x%p, desc %d, engine desc queued %d.\n", + transfer, transfer->desc_num, engine->desc_dequeued); + + head = list_entry(engine->transfer_list.next, struct xdma_transfer, + entry); + if (head == transfer) + list_del(engine->transfer_list.next); + else + pr_info("engine %s, transfer 0x%p NOT found, 0x%p.\n", + engine->name, transfer, head); + + if (transfer->state == TRANSFER_STATE_SUBMITTED) + transfer->state = TRANSFER_STATE_ABORTED; +} + +/* transfer_queue() - Queue a DMA transfer on the engine + * + * @engine DMA engine doing the transfer + * @transfer DMA transfer submitted to the engine + * + * Takes and releases the engine spinlock + */ +static int transfer_queue(struct xdma_engine *engine, + struct xdma_transfer *transfer) +{ + int rv = 0; + struct xdma_transfer *transfer_started; + struct xdma_dev *xdev; + unsigned long flags; + + BUG_ON(!engine); + BUG_ON(!engine->xdev); + BUG_ON(!transfer); + BUG_ON(transfer->desc_num == 0); + dbg_tfr("transfer_queue(transfer=0x%p).\n", transfer); + + xdev = engine->xdev; + if (xdma_device_flag_check(xdev, XDEV_FLAG_OFFLINE)) { + pr_info("dev 0x%p offline, transfer 0x%p not queued.\n", + xdev, transfer); + return -EBUSY; + } + + /* lock the engine state */ + spin_lock_irqsave(&engine->lock, flags); + + engine->prev_cpu = get_cpu(); + put_cpu(); + + /* engine is being shutdown; do not accept new transfers */ + if (engine->shutdown & ENGINE_SHUTDOWN_REQUEST) { + pr_info("engine %s offline, transfer 0x%p not queued.\n", + engine->name, transfer); + rv = -EBUSY; + goto shutdown; + } + + /* mark the transfer as submitted */ + transfer->state = TRANSFER_STATE_SUBMITTED; + /* add transfer to the tail of the engine transfer queue */ + list_add_tail(&transfer->entry, &engine->transfer_list); + + /* engine is idle? */ + if (!engine->running) { + /* start engine */ + dbg_tfr("transfer_queue(): starting %s engine.\n", + engine->name); + transfer_started = engine_start(engine); + dbg_tfr("transfer=0x%p started %s engine with transfer 0x%p.\n", + transfer, engine->name, transfer_started); + } else { + dbg_tfr("transfer=0x%p queued, with %s engine running.\n", + transfer, engine->name); + } + +shutdown: + /* unlock the engine state */ + dbg_tfr("engine->running = %d\n", engine->running); + spin_unlock_irqrestore(&engine->lock, flags); + return rv; +} + +static void engine_alignments(struct xdma_engine *engine) +{ + u32 w; + u32 align_bytes; + u32 granularity_bytes; + u32 address_bits; + + w = read_register(&engine->regs->alignments); + dbg_init("engine %p name %s alignments=0x%08x\n", engine, + engine->name, (int)w); + + /* RTO - add some macros to extract these fields */ + align_bytes = (w & 0x00ff0000U) >> 16; + granularity_bytes = (w & 0x0000ff00U) >> 8; + address_bits = (w & 0x000000ffU); + + dbg_init("align_bytes = %d\n", align_bytes); + dbg_init("granularity_bytes = %d\n", granularity_bytes); + dbg_init("address_bits = %d\n", address_bits); + + if (w) { + engine->addr_align = align_bytes; + engine->len_granularity = granularity_bytes; + engine->addr_bits = address_bits; + } else { + /* Some default values if alignments are unspecified */ + engine->addr_align = 1; + engine->len_granularity = 1; + engine->addr_bits = 64; + } +} + +static void engine_free_resource(struct xdma_engine *engine) +{ + struct xdma_dev *xdev = engine->xdev; + + /* Release memory use for descriptor writebacks */ + if (engine->poll_mode_addr_virt) { + dbg_sg("Releasing memory for descriptor writeback\n"); + dma_free_coherent(&xdev->pdev->dev, + sizeof(struct xdma_poll_wb), + engine->poll_mode_addr_virt, + engine->poll_mode_bus); + dbg_sg("Released memory for descriptor writeback\n"); + engine->poll_mode_addr_virt = NULL; + } + + if (engine->desc) { + dbg_init("device %s, engine %s pre-alloc desc 0x%p,0x%llx.\n", + dev_name(&xdev->pdev->dev), engine->name, + engine->desc, engine->desc_bus); + dma_free_coherent(&xdev->pdev->dev, + XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc), + engine->desc, engine->desc_bus); + engine->desc = NULL; + } + + if (engine->cyclic_result) { + dma_free_coherent(&xdev->pdev->dev, + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result), + engine->cyclic_result, engine->cyclic_result_bus); + engine->cyclic_result = NULL; + } +} + +static void engine_destroy(struct xdma_dev *xdev, struct xdma_engine *engine) +{ + BUG_ON(!xdev); + BUG_ON(!engine); + + dbg_sg("Shutting down engine %s%d", engine->name, engine->channel); + + /* Disable interrupts to stop processing new events during shutdown */ + write_register(0x0, &engine->regs->interrupt_enable_mask, + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); + + if (enable_credit_mp && engine->streaming && + engine->dir == DMA_FROM_DEVICE) { + u32 reg_value = (0x1 << engine->channel) << 16; + struct sgdma_common_regs *reg = (struct sgdma_common_regs *) + (xdev->bar[xdev->config_bar_idx] + + (0x6*TARGET_SPACING)); + write_register(reg_value, ®->credit_mode_enable_w1c, 0); + } + + /* Release memory use for descriptor writebacks */ + engine_free_resource(engine); + + memset(engine, 0, sizeof(struct xdma_engine)); + /* Decrement the number of engines available */ + xdev->engines_num--; +} + +/** + *engine_cyclic_stop() - stop a cyclic transfer running on an SG DMA engine + * + *engine->lock must be taken + */ +struct xdma_transfer *engine_cyclic_stop(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer = 0; + + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) { + /* pick first transfer on the queue (was submitted to engine) */ + transfer = list_entry(engine->transfer_list.next, + struct xdma_transfer, entry); + BUG_ON(!transfer); + + xdma_engine_stop(engine); + + if (transfer->cyclic) { + if (engine->xdma_perf) + dbg_perf("Stopping perf transfer on %s\n", + engine->name); + else + dbg_perf("Stopping cyclic transfer on %s\n", + engine->name); + /* make sure the handler sees correct transfer state */ + transfer->cyclic = 1; + /* + * set STOP flag and interrupt on completion, on the + * last descriptor + */ + xdma_desc_control_set( + transfer->desc_virt + transfer->desc_num - 1, + XDMA_DESC_COMPLETED | XDMA_DESC_STOPPED); + } else { + dbg_sg("(engine=%p) running transfer is not cyclic\n", + engine); + } + } else { + dbg_sg("(engine=%p) found not running transfer.\n", engine); + } + return transfer; +} +EXPORT_SYMBOL_GPL(engine_cyclic_stop); + +static int engine_writeback_setup(struct xdma_engine *engine) +{ + u32 w; + struct xdma_dev *xdev; + struct xdma_poll_wb *writeback; + + BUG_ON(!engine); + xdev = engine->xdev; + BUG_ON(!xdev); + + /* + * RTO - doing the allocation per engine is wasteful since a full page + * is allocated each time - better to allocate one page for the whole + * device during probe() and set per-engine offsets here + */ + writeback = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + writeback->completed_desc_count = 0; + + dbg_init("Setting writeback location to 0x%llx for engine %p", + engine->poll_mode_bus, engine); + w = cpu_to_le32(PCI_DMA_L(engine->poll_mode_bus)); + write_register(w, &engine->regs->poll_mode_wb_lo, + (unsigned long)(&engine->regs->poll_mode_wb_lo) - + (unsigned long)(&engine->regs)); + w = cpu_to_le32(PCI_DMA_H(engine->poll_mode_bus)); + write_register(w, &engine->regs->poll_mode_wb_hi, + (unsigned long)(&engine->regs->poll_mode_wb_hi) - + (unsigned long)(&engine->regs)); + + return 0; +} + + +/* engine_create() - Create an SG DMA engine bookkeeping data structure + * + * An SG DMA engine consists of the resources for a single-direction transfer + * queue; the SG DMA hardware, the software queue and interrupt handling. + * + * @dev Pointer to pci_dev + * @offset byte address offset in BAR[xdev->config_bar_idx] resource for the + * SG DMA * controller registers. + * @dir: DMA_TO/FROM_DEVICE + * @streaming Whether the engine is attached to AXI ST (rather than MM) + */ +static int engine_init_regs(struct xdma_engine *engine) +{ + u32 reg_value; + int rv = 0; + + write_register(XDMA_CTRL_NON_INCR_ADDR, &engine->regs->control_w1c, + (unsigned long)(&engine->regs->control_w1c) - + (unsigned long)(&engine->regs)); + + engine_alignments(engine); + + /* Configure error interrupts by default */ + reg_value = XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + reg_value |= XDMA_CTRL_IE_MAGIC_STOPPED; + reg_value |= XDMA_CTRL_IE_MAGIC_STOPPED; + reg_value |= XDMA_CTRL_IE_READ_ERROR; + reg_value |= XDMA_CTRL_IE_DESC_ERROR; + + /* if using polled mode, configure writeback address */ + if (poll_mode) { + rv = engine_writeback_setup(engine); + if (rv) { + dbg_init("%s descr writeback setup failed.\n", + engine->name); + goto fail_wb; + } + } else { + /* enable the relevant completion interrupts */ + reg_value |= XDMA_CTRL_IE_DESC_STOPPED; + reg_value |= XDMA_CTRL_IE_DESC_COMPLETED; + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) + reg_value |= XDMA_CTRL_IE_IDLE_STOPPED; + } + + /* Apply engine configurations */ + write_register(reg_value, &engine->regs->interrupt_enable_mask, + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); + + engine->interrupt_enable_mask_value = reg_value; + + /* only enable credit mode for AXI-ST C2H */ + if (enable_credit_mp && engine->streaming && + engine->dir == DMA_FROM_DEVICE) { + + struct xdma_dev *xdev = engine->xdev; + u32 reg_value = (0x1 << engine->channel) << 16; + struct sgdma_common_regs *reg = (struct sgdma_common_regs *) + (xdev->bar[xdev->config_bar_idx] + + (0x6*TARGET_SPACING)); + + write_register(reg_value, ®->credit_mode_enable_w1s, 0); + } + + return 0; + +fail_wb: + return rv; +} + +static int engine_alloc_resource(struct xdma_engine *engine) +{ + struct xdma_dev *xdev = engine->xdev; + + engine->desc = dma_alloc_coherent(&xdev->pdev->dev, + XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc), + &engine->desc_bus, GFP_KERNEL); + if (!engine->desc) { + pr_warn("dev %s, %s pre-alloc desc OOM.\n", + dev_name(&xdev->pdev->dev), engine->name); + goto err_out; + } + + if (poll_mode) { + engine->poll_mode_addr_virt = dma_alloc_coherent( + &xdev->pdev->dev, + sizeof(struct xdma_poll_wb), + &engine->poll_mode_bus, GFP_KERNEL); + if (!engine->poll_mode_addr_virt) { + pr_warn("%s, %s poll pre-alloc writeback OOM.\n", + dev_name(&xdev->pdev->dev), engine->name); + goto err_out; + } + } + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { + engine->cyclic_result = dma_alloc_coherent(&xdev->pdev->dev, + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result), + &engine->cyclic_result_bus, GFP_KERNEL); + + if (!engine->cyclic_result) { + pr_warn("%s, %s pre-alloc result OOM.\n", + dev_name(&xdev->pdev->dev), engine->name); + goto err_out; + } + } + + return 0; + +err_out: + engine_free_resource(engine); + return -ENOMEM; +} + +static int engine_init(struct xdma_engine *engine, struct xdma_dev *xdev, + int offset, enum dma_data_direction dir, int channel) +{ + int rv; + u32 val; + + dbg_init("channel %d, offset 0x%x, dir %d.\n", channel, offset, dir); + + /* set magic */ + engine->magic = MAGIC_ENGINE; + + engine->channel = channel; + + /* engine interrupt request bit */ + engine->irq_bitmask = (1 << XDMA_ENG_IRQ_NUM) - 1; + engine->irq_bitmask <<= (xdev->engines_num * XDMA_ENG_IRQ_NUM); + engine->bypass_offset = xdev->engines_num * BYPASS_MODE_SPACING; + + /* parent */ + engine->xdev = xdev; + /* register address */ + engine->regs = (xdev->bar[xdev->config_bar_idx] + offset); + engine->sgdma_regs = xdev->bar[xdev->config_bar_idx] + offset + + SGDMA_OFFSET_FROM_CHANNEL; + val = read_register(&engine->regs->identifier); + if (val & 0x8000U) + engine->streaming = 1; + + /* remember SG DMA direction */ + engine->dir = dir; + sprintf(engine->name, "%d-%s%d-%s", xdev->idx, + (dir == DMA_TO_DEVICE) ? "H2C" : "C2H", channel, + engine->streaming ? "ST" : "MM"); + + dbg_init("engine %p name %s irq_bitmask=0x%08x\n", engine, engine->name, + (int)engine->irq_bitmask); + + /* initialize the deferred work for transfer completion */ + INIT_WORK(&engine->work, engine_service_work); + + if (dir == DMA_TO_DEVICE) + xdev->mask_irq_h2c |= engine->irq_bitmask; + else + xdev->mask_irq_c2h |= engine->irq_bitmask; + xdev->engines_num++; + + rv = engine_alloc_resource(engine); + if (rv) + return rv; + + rv = engine_init_regs(engine); + if (rv) + return rv; + + return 0; +} + +/* transfer_destroy() - free transfer */ +static void transfer_destroy(struct xdma_dev *xdev, struct xdma_transfer *xfer) +{ + /* free descriptors */ + xdma_desc_done(xfer->desc_virt); + + if (xfer->last_in_request && (xfer->flags & XFER_FLAG_NEED_UNMAP)) { + struct sg_table *sgt = xfer->sgt; + + if (sgt->nents) { + pci_unmap_sg(xdev->pdev, sgt->sgl, sgt->nents, + xfer->dir); + sgt->nents = 0; + } + } +} + +static int transfer_build(struct xdma_engine *engine, + struct xdma_request_cb *req, unsigned int desc_max) +{ + struct xdma_transfer *xfer = &req->xfer; + struct sw_desc *sdesc = &(req->sdesc[req->sw_desc_idx]); + int i = 0; + int j = 0; + + for (; i < desc_max; i++, j++, sdesc++) { + dbg_desc("sw desc %d/%u: 0x%llx, 0x%x, ep 0x%llx.\n", + i + req->sw_desc_idx, req->sw_desc_cnt, + sdesc->addr, sdesc->len, req->ep_addr); + + /* fill in descriptor entry j with transfer details */ + xdma_desc_set(xfer->desc_virt + j, sdesc->addr, req->ep_addr, + sdesc->len, xfer->dir); + xfer->len += sdesc->len; + + /* for non-inc-add mode don't increment ep_addr */ + if (!engine->non_incr_addr) + req->ep_addr += sdesc->len; + } + req->sw_desc_idx += desc_max; + return 0; +} + +static int transfer_init(struct xdma_engine *engine, struct xdma_request_cb *req) +{ + struct xdma_transfer *xfer = &req->xfer; + unsigned int desc_max = min_t(unsigned int, + req->sw_desc_cnt - req->sw_desc_idx, + XDMA_TRANSFER_MAX_DESC); + int i = 0; + int last = 0; + u32 control; + + memset(xfer, 0, sizeof(*xfer)); + + /* initialize wait queue */ + init_waitqueue_head(&xfer->wq); + + /* remember direction of transfer */ + xfer->dir = engine->dir; + + xfer->desc_virt = engine->desc; + xfer->desc_bus = engine->desc_bus; + + transfer_desc_init(xfer, desc_max); + + dbg_sg("transfer->desc_bus = 0x%llx.\n", (u64)xfer->desc_bus); + + transfer_build(engine, req, desc_max); + + /* terminate last descriptor */ + last = desc_max - 1; + xdma_desc_link(xfer->desc_virt + last, 0, 0); + /* stop engine, EOP for AXI ST, req IRQ on last descriptor */ + control = XDMA_DESC_STOPPED; + control |= XDMA_DESC_EOP; + control |= XDMA_DESC_COMPLETED; + xdma_desc_control_set(xfer->desc_virt + last, control); + + xfer->desc_num = xfer->desc_adjacent = desc_max; + + dbg_sg("transfer 0x%p has %d descriptors\n", xfer, xfer->desc_num); + /* fill in adjacent numbers */ + for (i = 0; i < xfer->desc_num; i++) + xdma_desc_adjacent(xfer->desc_virt + i, xfer->desc_num - i - 1); + + return 0; +} + +#ifdef __LIBXDMA_DEBUG__ +static void sgt_dump(struct sg_table *sgt) +{ + int i; + struct scatterlist *sg = sgt->sgl; + + pr_info("sgt 0x%p, sgl 0x%p, nents %u/%u.\n", + sgt, sgt->sgl, sgt->nents, sgt->orig_nents); + + for (i = 0; i < sgt->orig_nents; i++, sg = sg_next(sg)) + pr_info("%d, 0x%p, pg 0x%p,%u+%u, dma 0x%llx,%u.\n", + i, sg, sg_page(sg), sg->offset, sg->length, + sg_dma_address(sg), sg_dma_len(sg)); +} + +static void xdma_request_cb_dump(struct xdma_request_cb *req) +{ + int i; + + pr_info("request 0x%p, total %u, ep 0x%llx, sw_desc %u, sgt 0x%p.\n", + req, req->total_len, req->ep_addr, req->sw_desc_cnt, req->sgt); + sgt_dump(req->sgt); + for (i = 0; i < req->sw_desc_cnt; i++) + pr_info("%d/%u, 0x%llx, %u.\n", + i, req->sw_desc_cnt, req->sdesc[i].addr, + req->sdesc[i].len); +} +#endif + +static void xdma_request_free(struct xdma_request_cb *req) +{ + if (((unsigned long)req) >= VMALLOC_START && + ((unsigned long)req) < VMALLOC_END) + vfree(req); + else + kfree(req); +} + +static struct xdma_request_cb * xdma_request_alloc(unsigned int sdesc_nr) +{ + struct xdma_request_cb *req; + unsigned int size = sizeof(struct xdma_request_cb) + + sdesc_nr * sizeof(struct sw_desc); + + req = kzalloc(size, GFP_KERNEL); + if (!req) { + req = vmalloc(size); + if (req) + memset(req, 0, size); + } + if (!req) { + pr_info("OOM, %u sw_desc, %u.\n", sdesc_nr, size); + return NULL; + } + + return req; +} + +static struct xdma_request_cb * xdma_init_request(struct sg_table *sgt, + u64 ep_addr) +{ + struct xdma_request_cb *req; + struct scatterlist *sg = sgt->sgl; + int max = sgt->nents; + int extra = 0; + int i, j = 0; + + for (i = 0; i < max; i++, sg = sg_next(sg)) { + unsigned int len = sg_dma_len(sg); + + if (unlikely(len > desc_blen_max)) + extra += (len + desc_blen_max - 1) / desc_blen_max; + } + +//pr_info("ep 0x%llx, desc %u+%u.\n", ep_addr, max, extra); + + max += extra; + req = xdma_request_alloc(max); + if (!req) + return NULL; + + req->sgt = sgt; + req->ep_addr = ep_addr; + + for (i = 0, sg = sgt->sgl; i < sgt->nents; i++, sg = sg_next(sg)) { + unsigned int tlen = sg_dma_len(sg); + dma_addr_t addr = sg_dma_address(sg); + + req->total_len += tlen; + while (tlen) { + req->sdesc[j].addr = addr; + if (tlen > desc_blen_max) { + req->sdesc[j].len = desc_blen_max; + addr += desc_blen_max; + tlen -= desc_blen_max; + } else { + req->sdesc[j].len = tlen; + tlen = 0; + } + j++; + } + } + BUG_ON(j > max); + + req->sw_desc_cnt = j; +#ifdef __LIBXDMA_DEBUG__ + xdma_request_cb_dump(req); +#endif + return req; +} + +ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, + struct sg_table *sgt, bool dma_mapped, int timeout_ms) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + struct xdma_engine *engine; + int rv = 0; + ssize_t done = 0; + struct scatterlist *sg = sgt->sgl; + int nents; + enum dma_data_direction dir = write ? DMA_TO_DEVICE : DMA_FROM_DEVICE; + struct xdma_request_cb *req = NULL; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + if (write == 1) { + if (channel >= xdev->h2c_channel_max) { + pr_warn("H2C channel %d >= %d.\n", + channel, xdev->h2c_channel_max); + return -EINVAL; + } + engine = &xdev->engine_h2c[channel]; + } else if (write == 0) { + if (channel >= xdev->c2h_channel_max) { + pr_warn("C2H channel %d >= %d.\n", + channel, xdev->c2h_channel_max); + return -EINVAL; + } + engine = &xdev->engine_c2h[channel]; + } else { + pr_warn("write %d, exp. 0|1.\n", write); + return -EINVAL; + } + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + xdev = engine->xdev; + if (xdma_device_flag_check(xdev, XDEV_FLAG_OFFLINE)) { + pr_info("xdev 0x%p, offline.\n", xdev); + return -EBUSY; + } + + /* check the direction */ + if (engine->dir != dir) { + pr_info("0x%p, %s, %d, W %d, 0x%x/0x%x mismatch.\n", + engine, engine->name, channel, write, engine->dir, dir); + return -EINVAL; + } + + if (!dma_mapped) { + nents = pci_map_sg(xdev->pdev, sg, sgt->orig_nents, dir); + if (!nents) { + pr_info("map sgl failed, sgt 0x%p.\n", sgt); + return -EIO; + } + sgt->nents = nents; + } else { + BUG_ON(!sgt->nents); + } + + req = xdma_init_request(sgt, ep_addr); + if (!req) { + rv = -ENOMEM; + goto unmap_sgl; + } + + dbg_tfr("%s, len %u sg cnt %u.\n", + engine->name, req->total_len, req->sw_desc_cnt); + + sg = sgt->sgl; + nents = req->sw_desc_cnt; + while (nents) { + unsigned long flags; + struct xdma_transfer *xfer; + + /* one transfer at a time */ + spin_lock(&engine->desc_lock); + + /* build transfer */ + rv = transfer_init(engine, req); + if (rv < 0) { + spin_unlock(&engine->desc_lock); + goto unmap_sgl; + } + xfer = &req->xfer; + + if (!dma_mapped) + xfer->flags = XFER_FLAG_NEED_UNMAP; + + /* last transfer for the given request? */ + nents -= xfer->desc_num; + if (!nents) { + xfer->last_in_request = 1; + xfer->sgt = sgt; + } + + dbg_tfr("xfer, %u, ep 0x%llx, done %lu, sg %u/%u.\n", + xfer->len, req->ep_addr, done, req->sw_desc_idx, + req->sw_desc_cnt); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); +#endif + + rv = transfer_queue(engine, xfer); + if (rv < 0) { + spin_unlock(&engine->desc_lock); + pr_info("unable to submit %s, %d.\n", engine->name, rv); + goto unmap_sgl; + } + + /* + * When polling, determine how many descriptors have been queued * on the engine to determine the writeback value expected + */ + if (poll_mode) { + unsigned int desc_count; + + spin_lock_irqsave(&engine->lock, flags); + desc_count = xfer->desc_num; + spin_unlock_irqrestore(&engine->lock, flags); + + dbg_tfr("%s poll desc_count=%d\n", + engine->name, desc_count); + rv = engine_service_poll(engine, desc_count); + + } else { + rv = wait_event_interruptible_timeout(xfer->wq, + (xfer->state != TRANSFER_STATE_SUBMITTED), + msecs_to_jiffies(timeout_ms)); + } + + spin_lock_irqsave(&engine->lock, flags); + + switch(xfer->state) { + case TRANSFER_STATE_COMPLETED: + spin_unlock_irqrestore(&engine->lock, flags); + + dbg_tfr("transfer %p, %u, ep 0x%llx compl, +%lu.\n", + xfer, xfer->len, req->ep_addr - xfer->len, done); + done += xfer->len; + rv = 0; + break; + case TRANSFER_STATE_FAILED: + pr_info("xfer 0x%p,%u, failed, ep 0x%llx.\n", + xfer, xfer->len, req->ep_addr - xfer->len); + spin_unlock_irqrestore(&engine->lock, flags); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); + sgt_dump(sgt); +#endif + rv = -EIO; + break; + default: + /* transfer can still be in-flight */ + pr_info("xfer 0x%p,%u, s 0x%x timed out, ep 0x%llx.\n", + xfer, xfer->len, xfer->state, req->ep_addr); + engine_status_read(engine, 0, 1); + //engine_status_dump(engine); + transfer_abort(engine, xfer); + + xdma_engine_stop(engine); + spin_unlock_irqrestore(&engine->lock, flags); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); + sgt_dump(sgt); +#endif + rv = -ERESTARTSYS; + break; + } + + transfer_destroy(xdev, xfer); + spin_unlock(&engine->desc_lock); + + if (rv < 0) + goto unmap_sgl; + } /* while (sg) */ + +unmap_sgl: + if (!dma_mapped && sgt->nents) { + pci_unmap_sg(xdev->pdev, sgt->sgl, sgt->orig_nents, dir); + sgt->nents = 0; + } + + if (req) + xdma_request_free(req); + + if (rv < 0) + return rv; + + return done; +} +EXPORT_SYMBOL_GPL(xdma_xfer_submit); + +int xdma_performance_submit(struct xdma_dev *xdev, struct xdma_engine *engine) +{ + u8 *buffer_virt; + u32 max_consistent_size = 128 * 32 * 1024; /* 1024 pages, 4MB */ + dma_addr_t buffer_bus; /* bus address */ + struct xdma_transfer *transfer; + u64 ep_addr = 0; + int num_desc_in_a_loop = 128; + int size_in_desc = engine->xdma_perf->transfer_size; + int size = size_in_desc * num_desc_in_a_loop; + int i; + + BUG_ON(size_in_desc > max_consistent_size); + + if (size > max_consistent_size) { + size = max_consistent_size; + num_desc_in_a_loop = size / size_in_desc; + } + + buffer_virt = dma_alloc_coherent(&xdev->pdev->dev, size, + &buffer_bus, GFP_KERNEL); + + /* allocate transfer data structure */ + transfer = kzalloc(sizeof(struct xdma_transfer), GFP_KERNEL); + BUG_ON(!transfer); + + /* 0 = write engine (to_dev=0) , 1 = read engine (to_dev=1) */ + transfer->dir = engine->dir; + /* set number of descriptors */ + transfer->desc_num = num_desc_in_a_loop; + + /* allocate descriptor list */ + if (!engine->desc) { + engine->desc = dma_alloc_coherent(&xdev->pdev->dev, + num_desc_in_a_loop * sizeof(struct xdma_desc), + &engine->desc_bus, GFP_KERNEL); + BUG_ON(!engine->desc); + dbg_init("device %s, engine %s pre-alloc desc 0x%p,0x%llx.\n", + dev_name(&xdev->pdev->dev), engine->name, + engine->desc, engine->desc_bus); + } + transfer->desc_virt = engine->desc; + transfer->desc_bus = engine->desc_bus; + + transfer_desc_init(transfer, transfer->desc_num); + + dbg_sg("transfer->desc_bus = 0x%llx.\n", (u64)transfer->desc_bus); + + for (i = 0; i < transfer->desc_num; i++) { + struct xdma_desc *desc = transfer->desc_virt + i; + dma_addr_t rc_bus_addr = buffer_bus + size_in_desc * i; + + /* fill in descriptor entry with transfer details */ + xdma_desc_set(desc, rc_bus_addr, ep_addr, size_in_desc, + engine->dir); + } + + /* stop engine and request interrupt on last descriptor */ + xdma_desc_control_set(transfer->desc_virt, 0); + /* create a linked loop */ + xdma_desc_link(transfer->desc_virt + transfer->desc_num - 1, + transfer->desc_virt, transfer->desc_bus); + + transfer->cyclic = 1; + + /* initialize wait queue */ + init_waitqueue_head(&transfer->wq); + + //printk("=== Descriptor print for PERF \n"); + //transfer_dump(transfer); + + dbg_perf("Queueing XDMA I/O %s request for performance measurement.\n", + engine->dir ? "write (to dev)" : "read (from dev)"); + transfer_queue(engine, transfer); + return 0; + +} +EXPORT_SYMBOL_GPL(xdma_performance_submit); + +static struct xdma_dev *alloc_dev_instance(struct pci_dev *pdev) +{ + int i; + struct xdma_dev *xdev; + struct xdma_engine *engine; + + BUG_ON(!pdev); + + /* allocate zeroed device book keeping structure */ + xdev = kzalloc(sizeof(struct xdma_dev), GFP_KERNEL); + if (!xdev) { + pr_info("OOM, xdma_dev.\n"); + return NULL; + } + spin_lock_init(&xdev->lock); + + xdev->magic = MAGIC_DEVICE; + xdev->config_bar_idx = -1; + xdev->user_bar_idx = -1; + xdev->bypass_bar_idx = -1; + xdev->irq_line = -1; + + /* create a driver to device reference */ + xdev->pdev = pdev; + dbg_init("xdev = 0x%p\n", xdev); + + /* Set up data user IRQ data structures */ + for (i = 0; i < 16; i++) { + xdev->user_irq[i].xdev = xdev; + spin_lock_init(&xdev->user_irq[i].events_lock); + init_waitqueue_head(&xdev->user_irq[i].events_wq); + xdev->user_irq[i].handler = NULL; + xdev->user_irq[i].user_idx = i; /* 0 based */ + } + + engine = xdev->engine_h2c; + for (i = 0; i < XDMA_CHANNEL_NUM_MAX; i++, engine++) { + spin_lock_init(&engine->lock); + spin_lock_init(&engine->desc_lock); + INIT_LIST_HEAD(&engine->transfer_list); + init_waitqueue_head(&engine->shutdown_wq); + init_waitqueue_head(&engine->xdma_perf_wq); + } + + engine = xdev->engine_c2h; + for (i = 0; i < XDMA_CHANNEL_NUM_MAX; i++, engine++) { + spin_lock_init(&engine->lock); + spin_lock_init(&engine->desc_lock); + INIT_LIST_HEAD(&engine->transfer_list); + init_waitqueue_head(&engine->shutdown_wq); + init_waitqueue_head(&engine->xdma_perf_wq); + } + + return xdev; +} + +static int request_regions(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + int rv; + + BUG_ON(!xdev); + BUG_ON(!pdev); + + dbg_init("pci_request_regions()\n"); + rv = pci_request_regions(pdev, xdev->mod_name); + /* could not request all regions? */ + if (rv) { + dbg_init("pci_request_regions() = %d, device in use?\n", rv); + /* assume device is in use so do not disable it later */ + xdev->regions_in_use = 1; + } else { + xdev->got_regions = 1; + } + + return rv; +} + +static int set_dma_mask(struct pci_dev *pdev) +{ + BUG_ON(!pdev); + + dbg_init("sizeof(dma_addr_t) == %ld\n", sizeof(dma_addr_t)); + /* 64-bit addressing capability for XDMA? */ + if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + /* query for DMA transfer */ + /* @see Documentation/DMA-mapping.txt */ + dbg_init("pci_set_dma_mask()\n"); + /* use 64-bit DMA */ + dbg_init("Using a 64-bit DMA mask.\n"); + /* use 32-bit DMA for descriptors */ + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + /* use 64-bit DMA, 32-bit for consistent */ + } else if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) { + dbg_init("Could not set 64-bit DMA mask.\n"); + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + /* use 32-bit DMA */ + dbg_init("Using a 32-bit DMA mask.\n"); + } else { + dbg_init("No suitable DMA possible.\n"); + return -EINVAL; + } + + return 0; +} + +static u32 get_engine_channel_id(struct engine_regs *regs) +{ + u32 value; + + BUG_ON(!regs); + + value = read_register(®s->identifier); + + return (value & 0x00000f00U) >> 8; +} + +static u32 get_engine_id(struct engine_regs *regs) +{ + u32 value; + + BUG_ON(!regs); + + value = read_register(®s->identifier); + return (value & 0xffff0000U) >> 16; +} + +static void remove_engines(struct xdma_dev *xdev) +{ + struct xdma_engine *engine; + int i; + + BUG_ON(!xdev); + + /* iterate over channels */ + for (i = 0; i < xdev->h2c_channel_max; i++) { + engine = &xdev->engine_h2c[i]; + if (engine->magic == MAGIC_ENGINE) { + dbg_sg("Remove %s, %d", engine->name, i); + engine_destroy(xdev, engine); + dbg_sg("%s, %d removed", engine->name, i); + } + } + + for (i = 0; i < xdev->c2h_channel_max; i++) { + engine = &xdev->engine_c2h[i]; + if (engine->magic == MAGIC_ENGINE) { + dbg_sg("Remove %s, %d", engine->name, i); + engine_destroy(xdev, engine); + dbg_sg("%s, %d removed", engine->name, i); + } + } +} + +static int probe_for_engine(struct xdma_dev *xdev, enum dma_data_direction dir, + int channel) +{ + struct engine_regs *regs; + int offset = channel * CHANNEL_SPACING; + u32 engine_id; + u32 engine_id_expected; + u32 channel_id; + struct xdma_engine *engine; + int rv; + + /* register offset for the engine */ + /* read channels at 0x0000, write channels at 0x1000, + * channels at 0x100 interval */ + if (dir == DMA_TO_DEVICE) { + engine_id_expected = XDMA_ID_H2C; + engine = &xdev->engine_h2c[channel]; + } else { + offset += H2C_CHANNEL_OFFSET; + engine_id_expected = XDMA_ID_C2H; + engine = &xdev->engine_c2h[channel]; + } + + regs = xdev->bar[xdev->config_bar_idx] + offset; + engine_id = get_engine_id(regs); + channel_id = get_engine_channel_id(regs); + + if ((engine_id != engine_id_expected) || (channel_id != channel)) { + dbg_init("%s %d engine, reg off 0x%x, id mismatch 0x%x,0x%x," + "exp 0x%x,0x%x, SKIP.\n", + dir == DMA_TO_DEVICE ? "H2C" : "C2H", + channel, offset, engine_id, channel_id, + engine_id_expected, channel_id != channel); + return -EINVAL; + } + + dbg_init("found AXI %s %d engine, reg. off 0x%x, id 0x%x,0x%x.\n", + dir == DMA_TO_DEVICE ? "H2C" : "C2H", channel, + offset, engine_id, channel_id); + + /* allocate and initialize engine */ + rv = engine_init(engine, xdev, offset, dir, channel); + if (rv != 0) { + pr_info("failed to create AXI %s %d engine.\n", + dir == DMA_TO_DEVICE ? "H2C" : "C2H", + channel); + return rv; + } + + return 0; +} + +static int probe_engines(struct xdma_dev *xdev) +{ + int i; + int rv = 0; + + BUG_ON(!xdev); + + /* iterate over channels */ + for (i = 0; i < xdev->h2c_channel_max; i++) { + rv = probe_for_engine(xdev, DMA_TO_DEVICE, i); + if (rv) + break; + } + xdev->h2c_channel_max = i; + + for (i = 0; i < xdev->c2h_channel_max; i++) { + rv = probe_for_engine(xdev, DMA_FROM_DEVICE, i); + if (rv) + break; + } + xdev->c2h_channel_max = i; + + return 0; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0) +static void pci_enable_relaxed_ordering(struct pci_dev *pdev) +{ + pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN); +} +#else +static void pci_enable_relaxed_ordering(struct pci_dev *pdev) +{ + u16 v; + int pos; + + pos = pci_pcie_cap(pdev); + if (pos > 0) { + pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &v); + v |= PCI_EXP_DEVCTL_RELAX_EN; + pci_write_config_word(pdev, pos + PCI_EXP_DEVCTL, v); + } +} +#endif + +static void pci_check_extended_tag(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + u16 cap; + u32 v; + void *__iomem reg; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0) + pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &cap); +#else + int pos; + + pos = pci_pcie_cap(pdev); + if (pos > 0) + pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &cap); + else { + pr_info("pdev 0x%p, unable to access pcie cap.\n", pdev); + return; + } +#endif + + if ((cap & PCI_EXP_DEVCTL_EXT_TAG)) + return; + + /* extended tag not enabled */ + pr_info("0x%p EXT_TAG disabled.\n", pdev); + + if (xdev->config_bar_idx < 0) { + pr_info("pdev 0x%p, xdev 0x%p, config bar UNKNOWN.\n", + pdev, xdev); + return; + } + + reg = xdev->bar[xdev->config_bar_idx] + XDMA_OFS_CONFIG + 0x4C; + v = read_register(reg); + v = (v & 0xFF) | (((u32)32) << 8); + write_register(v, reg, XDMA_OFS_CONFIG + 0x4C); +} + +void *xdma_device_open(const char *mname, struct pci_dev *pdev, int *user_max, + int *h2c_channel_max, int *c2h_channel_max) +{ + struct xdma_dev *xdev = NULL; + int rv = 0; + + pr_info("%s device %s, 0x%p.\n", mname, dev_name(&pdev->dev), pdev); + + /* allocate zeroed device book keeping structure */ + xdev = alloc_dev_instance(pdev); + if (!xdev) + return NULL; + xdev->mod_name = mname; + xdev->user_max = *user_max; + xdev->h2c_channel_max = *h2c_channel_max; + xdev->c2h_channel_max = *c2h_channel_max; + + xdma_device_flag_set(xdev, XDEV_FLAG_OFFLINE); + xdev_list_add(xdev); + + if (xdev->user_max == 0 || xdev->user_max > MAX_USER_IRQ) + xdev->user_max = MAX_USER_IRQ; + if (xdev->h2c_channel_max == 0 || + xdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX) + xdev->h2c_channel_max = XDMA_CHANNEL_NUM_MAX; + if (xdev->c2h_channel_max == 0 || + xdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX) + xdev->c2h_channel_max = XDMA_CHANNEL_NUM_MAX; + + rv = pci_enable_device(pdev); + if (rv) { + dbg_init("pci_enable_device() failed, %d.\n", rv); + goto err_enable; + } + + /* keep INTx enabled */ + pci_check_intr_pend(pdev); + + /* enable relaxed ordering */ + pci_enable_relaxed_ordering(pdev); + + pci_check_extended_tag(xdev, pdev); + + /* force MRRS to be 512 */ + rv = pcie_set_readrq(pdev, 512); + if (rv) + pr_info("device %s, error set PCI_EXP_DEVCTL_READRQ: %d.\n", + dev_name(&pdev->dev), rv); + + /* enable bus master capability */ + pci_set_master(pdev); + + rv = request_regions(xdev, pdev); + if (rv) + goto err_regions; + + rv = map_bars(xdev, pdev); + if (rv) + goto err_map; + + rv = set_dma_mask(pdev); + if (rv) + goto err_mask; + + check_nonzero_interrupt_status(xdev); + /* explicitely zero all interrupt enable masks */ + channel_interrupts_disable(xdev, ~0); + user_interrupts_disable(xdev, ~0); + read_interrupts(xdev); + + rv = probe_engines(xdev); + if (rv) + goto err_engines; + + rv = enable_msi_msix(xdev, pdev); + if (rv < 0) + goto err_enable_msix; + + rv = irq_setup(xdev, pdev); + if (rv < 0) + goto err_interrupts; + + if (!poll_mode) + channel_interrupts_enable(xdev, ~0); + + /* Flush writes */ + read_interrupts(xdev); + + *user_max = xdev->user_max; + *h2c_channel_max = xdev->h2c_channel_max; + *c2h_channel_max = xdev->c2h_channel_max; + + xdma_device_flag_clear(xdev, XDEV_FLAG_OFFLINE); + return (void *)xdev; + +err_interrupts: + irq_teardown(xdev); +err_enable_msix: + disable_msi_msix(xdev, pdev); +err_engines: + remove_engines(xdev); +err_mask: + unmap_bars(xdev, pdev); +err_map: + if (xdev->got_regions) + pci_release_regions(pdev); +err_regions: + if (!xdev->regions_in_use) + pci_disable_device(pdev); +err_enable: + xdev_list_remove(xdev); + kfree(xdev); + return NULL; +} +EXPORT_SYMBOL_GPL(xdma_device_open); + +void xdma_device_close(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + dbg_init("pdev 0x%p, xdev 0x%p.\n", pdev, dev_hndl); + + if (!dev_hndl) + return; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return; + + dbg_sg("remove(dev = 0x%p) where pdev->dev.driver_data = 0x%p\n", + pdev, xdev); + if (xdev->pdev != pdev) { + dbg_sg("pci_dev(0x%lx) != pdev(0x%lx)\n", + (unsigned long)xdev->pdev, (unsigned long)pdev); + } + + channel_interrupts_disable(xdev, ~0); + user_interrupts_disable(xdev, ~0); + read_interrupts(xdev); + + irq_teardown(xdev); + disable_msi_msix(xdev, pdev); + + remove_engines(xdev); + unmap_bars(xdev, pdev); + + if (xdev->got_regions) { + dbg_init("pci_release_regions 0x%p.\n", pdev); + pci_release_regions(pdev); + } + + if (!xdev->regions_in_use) { + dbg_init("pci_disable_device 0x%p.\n", pdev); + pci_disable_device(pdev); + } + + xdev_list_remove(xdev); + + kfree(xdev); +} +EXPORT_SYMBOL_GPL(xdma_device_close); + +void xdma_device_offline(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + struct xdma_engine *engine; + int i; + + if (!dev_hndl) + return; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return; + +pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); + xdma_device_flag_set(xdev, XDEV_FLAG_OFFLINE); + + /* wait for all engines to be idle */ + for (i = 0; i < xdev->h2c_channel_max; i++) { + unsigned long flags; + + engine = &xdev->engine_h2c[i]; + + if (engine->magic == MAGIC_ENGINE) { + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown |= ENGINE_SHUTDOWN_REQUEST; + + xdma_engine_stop(engine); + engine->running = 0; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + for (i = 0; i < xdev->c2h_channel_max; i++) { + unsigned long flags; + + engine = &xdev->engine_c2h[i]; + if (engine->magic == MAGIC_ENGINE) { + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown |= ENGINE_SHUTDOWN_REQUEST; + + xdma_engine_stop(engine); + engine->running = 0; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + /* turn off interrupts */ + channel_interrupts_disable(xdev, ~0); + user_interrupts_disable(xdev, ~0); + read_interrupts(xdev); + irq_teardown(xdev); + + pr_info("xdev 0x%p, done.\n", xdev); +} +EXPORT_SYMBOL_GPL(xdma_device_offline); + +void xdma_device_online(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + struct xdma_engine *engine; + unsigned long flags; + int i; + + if (!dev_hndl) + return; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return; + +pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); + + for (i = 0; i < xdev->h2c_channel_max; i++) { + engine = &xdev->engine_h2c[i]; + if (engine->magic == MAGIC_ENGINE) { + engine_init_regs(engine); + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown &= ~ENGINE_SHUTDOWN_REQUEST; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + for (i = 0; i < xdev->c2h_channel_max; i++) { + engine = &xdev->engine_c2h[i]; + if (engine->magic == MAGIC_ENGINE) { + engine_init_regs(engine); + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown &= ~ENGINE_SHUTDOWN_REQUEST; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + /* re-write the interrupt table */ + if (!poll_mode) { + irq_setup(xdev, pdev); + + channel_interrupts_enable(xdev, ~0); + user_interrupts_enable(xdev, xdev->mask_irq_user); + read_interrupts(xdev); + } + + xdma_device_flag_clear(xdev, XDEV_FLAG_OFFLINE); +pr_info("xdev 0x%p, done.\n", xdev); +} +EXPORT_SYMBOL_GPL(xdma_device_online); + +int xdma_device_restart(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return -EINVAL; + + pr_info("NOT implemented, 0x%p.\n", xdev); + return -EINVAL; +} +EXPORT_SYMBOL_GPL(xdma_device_restart); + +int xdma_user_isr_register(void *dev_hndl, unsigned int mask, + irq_handler_t handler, void *dev) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + int i; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + for (i = 0; i < xdev->user_max && mask; i++) { + unsigned int bit = (1 << i); + + if ((bit & mask) == 0) + continue; + + mask &= ~bit; + xdev->user_irq[i].handler = handler; + xdev->user_irq[i].dev = dev; + } + + return 0; +} +EXPORT_SYMBOL_GPL(xdma_user_isr_register); + +int xdma_user_isr_enable(void *dev_hndl, unsigned int mask) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + xdev->mask_irq_user |= mask; + /* enable user interrupts */ + user_interrupts_enable(xdev, mask); + read_interrupts(xdev); + + return 0; +} +EXPORT_SYMBOL_GPL(xdma_user_isr_enable); + +int xdma_user_isr_disable(void *dev_hndl, unsigned int mask) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + xdev->mask_irq_user &= ~mask; + user_interrupts_disable(xdev, mask); + read_interrupts(xdev); + + return 0; +} +EXPORT_SYMBOL_GPL(xdma_user_isr_disable); + +#ifdef __LIBXDMA_MOD__ +static int __init xdma_base_init(void) +{ + printk(KERN_INFO "%s", version); + return 0; +} + +static void __exit xdma_base_exit(void) +{ + return; +} + +module_init(xdma_base_init); +module_exit(xdma_base_exit); +#endif +/* makes an existing transfer cyclic */ +static void xdma_transfer_cyclic(struct xdma_transfer *transfer) +{ + /* link last descriptor to first descriptor */ + xdma_desc_link(transfer->desc_virt + transfer->desc_num - 1, + transfer->desc_virt, transfer->desc_bus); + /* remember transfer is cyclic */ + transfer->cyclic = 1; +} + +static int transfer_monitor_cyclic(struct xdma_engine *engine, + struct xdma_transfer *transfer, int timeout_ms) +{ + struct xdma_result *result; + int rc = 0; + + BUG_ON(!engine); + BUG_ON(!transfer); + + result = engine->cyclic_result; + BUG_ON(!result); + + if (poll_mode) { + int i ; + for (i = 0; i < 5; i++) { + rc = engine_service_poll(engine, 0); + if (rc) { + pr_info("%s service_poll failed %d.\n", + engine->name, rc); + rc = -ERESTARTSYS; + } + if (result[engine->rx_head].status) + return 0; + } + } else { + if (enable_credit_mp){ + dbg_tfr("%s: rx_head=%d,rx_tail=%d, wait ...\n", + engine->name, engine->rx_head, engine->rx_tail); + rc = wait_event_interruptible_timeout( transfer->wq, + (engine->rx_head!=engine->rx_tail || + engine->rx_overrun), + msecs_to_jiffies(timeout_ms)); + dbg_tfr("%s: wait returns %d, rx %d/%d, overrun %d.\n", + engine->name, rc, engine->rx_head, + engine->rx_tail, engine->rx_overrun); + } else { + rc = wait_event_interruptible_timeout( transfer->wq, + engine->eop_found, + msecs_to_jiffies(timeout_ms)); + dbg_tfr("%s: wait returns %d, eop_found %d.\n", + engine->name, rc, engine->eop_found); + } + } + + return 0; +} + +struct scatterlist *sglist_index(struct sg_table *sgt, unsigned int idx) +{ + struct scatterlist *sg = sgt->sgl; + int i; + + if (idx >= sgt->orig_nents) + return NULL; + + if (!idx) + return sg; + + for (i = 0; i < idx; i++, sg = sg_next(sg)) + ; + + return sg; +} + +static int copy_cyclic_to_user(struct xdma_engine *engine, int pkt_length, + int head, char __user *buf, size_t count) +{ + struct scatterlist *sg; + int more = pkt_length; + + BUG_ON(!engine); + BUG_ON(!buf); + + dbg_tfr("%s, pkt_len %d, head %d, user buf idx %u.\n", + engine->name, pkt_length, head, engine->user_buffer_index); + + sg = sglist_index(&engine->cyclic_sgt, head); + if (!sg) { + pr_info("%s, head %d OOR, sgl %u.\n", + engine->name, head, engine->cyclic_sgt.orig_nents); + return -EIO; + } + + /* EOP found? Transfer anything from head to EOP */ + while (more) { + unsigned int copy = more > PAGE_SIZE ? PAGE_SIZE : more; + unsigned int blen = count - engine->user_buffer_index; + int rv; + + if (copy > blen) + copy = blen; + + dbg_tfr("%s sg %d, 0x%p, copy %u to user %u.\n", + engine->name, head, sg, copy, + engine->user_buffer_index); + + rv = copy_to_user(&buf[engine->user_buffer_index], + page_address(sg_page(sg)), copy); + if (rv) { + pr_info("%s copy_to_user %u failed %d\n", + engine->name, copy, rv); + return -EIO; + } + + more -= copy; + engine->user_buffer_index += copy; + + if (engine->user_buffer_index == count) { + /* user buffer used up */ + break; + } + + head++; + if (head >= CYCLIC_RX_PAGES_MAX) { + head = 0; + sg = engine->cyclic_sgt.sgl; + } else + sg = sg_next(sg); + } + + return pkt_length; +} + +static int complete_cyclic(struct xdma_engine *engine, char __user *buf, + size_t count) +{ + struct xdma_result *result; + int pkt_length = 0; + int fault = 0; + int eop = 0; + int head; + int rc = 0; + int num_credit = 0; + unsigned long flags; + + BUG_ON(!engine); + result = engine->cyclic_result; + BUG_ON(!result); + + spin_lock_irqsave(&engine->lock, flags); + + /* where the host currently is in the ring buffer */ + head = engine->rx_head; + + /* iterate over newly received results */ + while (engine->rx_head != engine->rx_tail||engine->rx_overrun) { + + WARN_ON(result[engine->rx_head].status==0); + + dbg_tfr("%s, result[%d].status = 0x%x length = 0x%x.\n", + engine->name, engine->rx_head, + result[engine->rx_head].status, + result[engine->rx_head].length); + + if ((result[engine->rx_head].status >> 16) != C2H_WB) { + pr_info("%s, result[%d].status 0x%x, no magic.\n", + engine->name, engine->rx_head, + result[engine->rx_head].status); + fault = 1; + } else if (result[engine->rx_head].length > PAGE_SIZE) { + pr_info("%s, result[%d].len 0x%x, > PAGE_SIZE 0x%lx.\n", + engine->name, engine->rx_head, + result[engine->rx_head].length, PAGE_SIZE); + fault = 1; + } else if (result[engine->rx_head].length == 0) { + pr_info("%s, result[%d].length 0x%x.\n", + engine->name, engine->rx_head, + result[engine->rx_head].length); + fault = 1; + /* valid result */ + } else { + pkt_length += result[engine->rx_head].length; + num_credit++; + /* seen eop? */ + //if (result[engine->rx_head].status & RX_STATUS_EOP) + if (result[engine->rx_head].status & RX_STATUS_EOP){ + eop = 1; + engine->eop_found = 1; + } + + dbg_tfr("%s, pkt_length=%d (%s)\n", + engine->name, pkt_length, + eop ? "with EOP" : "no EOP yet"); + } + /* clear result */ + result[engine->rx_head].status = 0; + result[engine->rx_head].length = 0; + /* proceed head pointer so we make progress, even when fault */ + engine->rx_head = (engine->rx_head + 1) % CYCLIC_RX_PAGES_MAX; + + /* stop processing if a fault/eop was detected */ + if (fault || eop){ + break; + } + } + + spin_unlock_irqrestore(&engine->lock, flags); + + if (fault) + return -EIO; + + rc = copy_cyclic_to_user(engine, pkt_length, head, buf, count); + engine->rx_overrun = 0; + /* if copy is successful, release credits */ + if(rc > 0) + write_register(num_credit,&engine->sgdma_regs->credits, 0); + + return rc; +} + +ssize_t xdma_engine_read_cyclic(struct xdma_engine *engine, char __user *buf, + size_t count, int timeout_ms) +{ + int i = 0; + int rc = 0; + int rc_len = 0; + struct xdma_transfer *transfer; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + transfer = &engine->cyclic_req->xfer; + BUG_ON(!transfer); + + engine->user_buffer_index = 0; + + do { + rc = transfer_monitor_cyclic(engine, transfer, timeout_ms); + if (rc < 0) + return rc; + rc = complete_cyclic(engine, buf, count); + if (rc < 0) + return rc; + rc_len += rc; + + i++; + if (i > 10) + break; + } while (!engine->eop_found); + + if(enable_credit_mp) + engine->eop_found = 0; + + return rc_len; +} + +static void sgt_free_with_pages(struct sg_table *sgt, int dir, + struct pci_dev *pdev) +{ + struct scatterlist *sg = sgt->sgl; + int npages = sgt->orig_nents; + int i; + + for (i = 0; i < npages; i++, sg = sg_next(sg)) { + struct page *pg = sg_page(sg); + dma_addr_t bus = sg_dma_address(sg); + + if (pg) { + if (pdev) + pci_unmap_page(pdev, bus, PAGE_SIZE, dir); + __free_page(pg); + } else + break; + } + sg_free_table(sgt); + memset(sgt, 0, sizeof(struct sg_table)); +} + +static int sgt_alloc_with_pages(struct sg_table *sgt, unsigned int npages, + int dir, struct pci_dev *pdev) +{ + struct scatterlist *sg; + int i; + + if (sg_alloc_table(sgt, npages, GFP_KERNEL)) { + pr_info("sgt OOM.\n"); + return -ENOMEM; + } + + sg = sgt->sgl; + for (i = 0; i < npages; i++, sg = sg_next(sg)) { + struct page *pg = alloc_page(GFP_KERNEL); + + if (!pg) { + pr_info("%d/%u, page OOM.\n", i, npages); + goto err_out; + } + + if (pdev) { + dma_addr_t bus = pci_map_page(pdev, pg, 0, PAGE_SIZE, + dir); + if (unlikely(pci_dma_mapping_error(pdev, bus))) { + pr_info("%d/%u, page 0x%p map err.\n", + i, npages, pg); + __free_page(pg); + goto err_out; + } + sg_dma_address(sg) = bus; + sg_dma_len(sg) = PAGE_SIZE; + } + sg_set_page(sg, pg, PAGE_SIZE, 0); + } + + sgt->orig_nents = sgt->nents = npages; + + return 0; + +err_out: + sgt_free_with_pages(sgt, dir, pdev); + return -ENOMEM; +} + +/* + * !NOTE! reference/demo purpose only + * xdma_cyclic_transfer_setup is used for streaming C2H transfers: + * - A list of buffers are pre-allocated for incoming streaming data + * - the ring of the buffers is allowed to wrap around + */ +int xdma_cyclic_transfer_setup(struct xdma_engine *engine) +{ + struct xdma_dev *xdev; + struct xdma_transfer *xfer; + dma_addr_t bus; + unsigned long flags; + int i; + int rc; + + BUG_ON(!engine); + xdev = engine->xdev; + BUG_ON(!xdev); + + if (engine->cyclic_req) { + //pr_info("%s: exclusive access already taken.\n", + //engine->name); + return -EBUSY; + } + + spin_lock_irqsave(&engine->lock, flags); + + engine->rx_tail = 0; + engine->rx_head = 0; + engine->rx_overrun = 0; + engine->eop_found = 0; + + rc = sgt_alloc_with_pages(&engine->cyclic_sgt, CYCLIC_RX_PAGES_MAX, + engine->dir, xdev->pdev); + if (rc < 0) { + pr_info("%s cyclic pages %u OOM.\n", + engine->name, CYCLIC_RX_PAGES_MAX); + goto err_out; + } + + engine->cyclic_req = xdma_init_request(&engine->cyclic_sgt, 0); + if (!engine->cyclic_req) { + pr_info("%s cyclic request OOM.\n", engine->name); + rc = -ENOMEM; + goto err_out; + } + +#ifdef __LIBXDMA_DEBUG__ + xdma_request_cb_dump(engine->cyclic_req); +#endif + + rc = transfer_init(engine, engine->cyclic_req); + if (rc < 0) + goto err_out; + + xfer = &engine->cyclic_req->xfer; + + /* replace source addresses with result write-back addresses */ + memset(engine->cyclic_result, 0, + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result)); + bus = engine->cyclic_result_bus; + for (i = 0; i < xfer->desc_num; i++) { + xfer->desc_virt[i].src_addr_lo = cpu_to_le32(PCI_DMA_L(bus)); + xfer->desc_virt[i].src_addr_hi = cpu_to_le32(PCI_DMA_H(bus)); + bus += sizeof(struct xdma_result); + } + /* set control of all descriptors */ + for (i = 0; i < xfer->desc_num; i++) { + xdma_desc_control_clear(xfer->desc_virt + i, LS_BYTE_MASK); + xdma_desc_control_set(xfer->desc_virt + i, + XDMA_DESC_EOP | XDMA_DESC_COMPLETED); + } + + /* make this a cyclic transfer */ + xdma_transfer_cyclic(xfer); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); +#endif + + if(enable_credit_mp){ + //write_register(RX_BUF_PAGES,&engine->sgdma_regs->credits); + write_register(128, &engine->sgdma_regs->credits, 0); + } + + spin_unlock_irqrestore(&engine->lock, flags); + + /* start cyclic transfer */ + transfer_queue(engine, xfer); + + return 0; + + /* unwind on errors */ +err_out: + if (engine->cyclic_req) { + xdma_request_free(engine->cyclic_req); + engine->cyclic_req = NULL; + } + + if (engine->cyclic_sgt.orig_nents) { + sgt_free_with_pages(&engine->cyclic_sgt, engine->dir, + xdev->pdev); + engine->cyclic_sgt.orig_nents = 0; + engine->cyclic_sgt.nents = 0; + engine->cyclic_sgt.sgl = NULL; + } + + spin_unlock_irqrestore(&engine->lock, flags); + + return rc; +} + + +static int cyclic_shutdown_polled(struct xdma_engine *engine) +{ + BUG_ON(!engine); + + spin_lock(&engine->lock); + + dbg_tfr("Polling for shutdown completion\n"); + do { + engine_status_read(engine, 1, 0); + schedule(); + } while (engine->status & XDMA_STAT_BUSY); + + if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { + dbg_tfr("Engine has stopped\n"); + + if (!list_empty(&engine->transfer_list)) + engine_transfer_dequeue(engine); + + engine_service_shutdown(engine); + } + + dbg_tfr("Shutdown completion polling done\n"); + spin_unlock(&engine->lock); + + return 0; +} + +static int cyclic_shutdown_interrupt(struct xdma_engine *engine) +{ + int rc; + + BUG_ON(!engine); + + rc = wait_event_interruptible_timeout(engine->shutdown_wq, + !engine->running, msecs_to_jiffies(10000)); + +#if 0 + if (rc) { + dbg_tfr("wait_event_interruptible=%d\n", rc); + return rc; + } +#endif + + if (engine->running) { + pr_info("%s still running?!, %d\n", engine->name, rc); + return -EINVAL; + } + + return rc; +} + +int xdma_cyclic_transfer_teardown(struct xdma_engine *engine) +{ + int rc; + struct xdma_dev *xdev = engine->xdev; + struct xdma_transfer *transfer; + unsigned long flags; + + transfer = engine_cyclic_stop(engine); + + spin_lock_irqsave(&engine->lock, flags); + if (transfer) { + dbg_tfr("%s: stop transfer 0x%p.\n", engine->name, transfer); + if (transfer != &engine->cyclic_req->xfer) { + pr_info("%s unexpected transfer 0x%p/0x%p\n", + engine->name, transfer, + &engine->cyclic_req->xfer); + } + } + /* allow engine to be serviced after stop request */ + spin_unlock_irqrestore(&engine->lock, flags); + + /* wait for engine to be no longer running */ + if (poll_mode) + rc = cyclic_shutdown_polled(engine); + else + rc = cyclic_shutdown_interrupt(engine); + + /* obtain spin lock to atomically remove resources */ + spin_lock_irqsave(&engine->lock, flags); + + if (engine->cyclic_req) { + xdma_request_free(engine->cyclic_req); + engine->cyclic_req = NULL; + } + + if (engine->cyclic_sgt.orig_nents) { + sgt_free_with_pages(&engine->cyclic_sgt, engine->dir, + xdev->pdev); + engine->cyclic_sgt.orig_nents = 0; + engine->cyclic_sgt.nents = 0; + engine->cyclic_sgt.sgl = NULL; + } + + spin_unlock_irqrestore(&engine->lock, flags); + + return 0; +} + +int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg) +{ + int rv; + unsigned long dst; + u32 w = XDMA_CTRL_NON_INCR_ADDR; + + dbg_perf("IOCTL_XDMA_ADDRMODE_SET\n"); + rv = get_user(dst, (int __user *)arg); + + if (rv == 0) { + engine->non_incr_addr = !!dst; + if (engine->non_incr_addr) + write_register(w, &engine->regs->control_w1s, + (unsigned long)(&engine->regs->control_w1s) - + (unsigned long)(&engine->regs)); + else + write_register(w, &engine->regs->control_w1c, + (unsigned long)(&engine->regs->control_w1c) - + (unsigned long)(&engine->regs)); + } + engine_alignments(engine); + + return rv; +} + diff --git a/sources/xdma_driver/xdma/libxdma.h b/sources/xdma_driver/xdma/libxdma.h new file mode 100644 index 0000000..bbf1a45 --- /dev/null +++ b/sources/xdma_driver/xdma/libxdma.h @@ -0,0 +1,601 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef XDMA_LIB_H +#define XDMA_LIB_H + +#include <linux/version.h> +#include <linux/types.h> +#include <linux/uaccess.h> +#include <linux/module.h> +#include <linux/dma-mapping.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/jiffies.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/workqueue.h> + +/* Switch debug printing on/off */ +#define XDMA_DEBUG 0 + +/* SECTION: Preprocessor macros/constants */ +#define XDMA_BAR_NUM (6) + +/* maximum amount of register space to map */ +#define XDMA_BAR_SIZE (0x8000UL) + +/* Use this definition to poll several times between calls to schedule */ +#define NUM_POLLS_PER_SCHED 100 + +#define XDMA_CHANNEL_NUM_MAX (4) +/* + * interrupts per engine, rad2_vul.sv:237 + * .REG_IRQ_OUT (reg_irq_from_ch[(channel*2) +: 2]), + */ +#define XDMA_ENG_IRQ_NUM (1) +#define MAX_EXTRA_ADJ (15) +#define RX_STATUS_EOP (1) + +/* Target internal components on XDMA control BAR */ +#define XDMA_OFS_INT_CTRL (0x2000UL) +#define XDMA_OFS_CONFIG (0x3000UL) + +/* maximum number of desc per transfer request */ +#define XDMA_TRANSFER_MAX_DESC (2048) + +/* maximum size of a single DMA transfer descriptor */ +#define XDMA_DESC_BLEN_BITS 28 +#define XDMA_DESC_BLEN_MAX ((1 << (XDMA_DESC_BLEN_BITS)) - 1) + +/* bits of the SG DMA control register */ +#define XDMA_CTRL_RUN_STOP (1UL << 0) +#define XDMA_CTRL_IE_DESC_STOPPED (1UL << 1) +#define XDMA_CTRL_IE_DESC_COMPLETED (1UL << 2) +#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH (1UL << 3) +#define XDMA_CTRL_IE_MAGIC_STOPPED (1UL << 4) +#define XDMA_CTRL_IE_IDLE_STOPPED (1UL << 6) +#define XDMA_CTRL_IE_READ_ERROR (0x1FUL << 9) +#define XDMA_CTRL_IE_DESC_ERROR (0x1FUL << 19) +#define XDMA_CTRL_NON_INCR_ADDR (1UL << 25) +#define XDMA_CTRL_POLL_MODE_WB (1UL << 26) + +/* bits of the SG DMA status register */ +#define XDMA_STAT_BUSY (1UL << 0) +#define XDMA_STAT_DESC_STOPPED (1UL << 1) +#define XDMA_STAT_DESC_COMPLETED (1UL << 2) +#define XDMA_STAT_ALIGN_MISMATCH (1UL << 3) +#define XDMA_STAT_MAGIC_STOPPED (1UL << 4) +#define XDMA_STAT_INVALID_LEN (1UL << 5) +#define XDMA_STAT_IDLE_STOPPED (1UL << 6) + +#define XDMA_STAT_COMMON_ERR_MASK \ + (XDMA_STAT_ALIGN_MISMATCH | XDMA_STAT_MAGIC_STOPPED | \ + XDMA_STAT_INVALID_LEN) + +/* desc_error, C2H & H2C */ +#define XDMA_STAT_DESC_UNSUPP_REQ (1UL << 19) +#define XDMA_STAT_DESC_COMPL_ABORT (1UL << 20) +#define XDMA_STAT_DESC_PARITY_ERR (1UL << 21) +#define XDMA_STAT_DESC_HEADER_EP (1UL << 22) +#define XDMA_STAT_DESC_UNEXP_COMPL (1UL << 23) + +#define XDMA_STAT_DESC_ERR_MASK \ + (XDMA_STAT_DESC_UNSUPP_REQ | XDMA_STAT_DESC_COMPL_ABORT | \ + XDMA_STAT_DESC_PARITY_ERR | XDMA_STAT_DESC_HEADER_EP | \ + XDMA_STAT_DESC_UNEXP_COMPL) + +/* read error: H2C */ +#define XDMA_STAT_H2C_R_UNSUPP_REQ (1UL << 9) +#define XDMA_STAT_H2C_R_COMPL_ABORT (1UL << 10) +#define XDMA_STAT_H2C_R_PARITY_ERR (1UL << 11) +#define XDMA_STAT_H2C_R_HEADER_EP (1UL << 12) +#define XDMA_STAT_H2C_R_UNEXP_COMPL (1UL << 13) + +#define XDMA_STAT_H2C_R_ERR_MASK \ + (XDMA_STAT_H2C_R_UNSUPP_REQ | XDMA_STAT_H2C_R_COMPL_ABORT | \ + XDMA_STAT_H2C_R_PARITY_ERR | XDMA_STAT_H2C_R_HEADER_EP | \ + XDMA_STAT_H2C_R_UNEXP_COMPL) + +/* write error, H2C only */ +#define XDMA_STAT_H2C_W_DECODE_ERR (1UL << 14) +#define XDMA_STAT_H2C_W_SLAVE_ERR (1UL << 15) + +#define XDMA_STAT_H2C_W_ERR_MASK \ + (XDMA_STAT_H2C_W_DECODE_ERR | XDMA_STAT_H2C_W_SLAVE_ERR) + +/* read error: C2H */ +#define XDMA_STAT_C2H_R_DECODE_ERR (1UL << 9) +#define XDMA_STAT_C2H_R_SLAVE_ERR (1UL << 10) + +#define XDMA_STAT_C2H_R_ERR_MASK \ + (XDMA_STAT_C2H_R_DECODE_ERR | XDMA_STAT_C2H_R_SLAVE_ERR) + +/* all combined */ +#define XDMA_STAT_H2C_ERR_MASK \ + (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \ + XDMA_STAT_H2C_R_ERR_MASK | XDMA_STAT_H2C_W_ERR_MASK) + +#define XDMA_STAT_C2H_ERR_MASK \ + (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \ + XDMA_STAT_C2H_R_ERR_MASK) + +/* bits of the SGDMA descriptor control field */ +#define XDMA_DESC_STOPPED (1UL << 0) +#define XDMA_DESC_COMPLETED (1UL << 1) +#define XDMA_DESC_EOP (1UL << 4) + +#define XDMA_PERF_RUN (1UL << 0) +#define XDMA_PERF_CLEAR (1UL << 1) +#define XDMA_PERF_AUTO (1UL << 2) + +#define MAGIC_ENGINE 0xEEEEEEEEUL +#define MAGIC_DEVICE 0xDDDDDDDDUL + +/* upper 16-bits of engine identifier register */ +#define XDMA_ID_H2C 0x1fc0U +#define XDMA_ID_C2H 0x1fc1U + +/* for C2H AXI-ST mode */ +#define CYCLIC_RX_PAGES_MAX 256 + +#define LS_BYTE_MASK 0x000000FFUL + +#define BLOCK_ID_MASK 0xFFF00000 +#define BLOCK_ID_HEAD 0x1FC00000 + +#define IRQ_BLOCK_ID 0x1fc20000UL +#define CONFIG_BLOCK_ID 0x1fc30000UL + +#define WB_COUNT_MASK 0x00ffffffUL +#define WB_ERR_MASK (1UL << 31) +#define POLL_TIMEOUT_SECONDS 10 + +#define MAX_USER_IRQ 16 + +#define MAX_DESC_BUS_ADDR (0xffffffffULL) + +#define DESC_MAGIC 0xAD4B0000UL + +#define C2H_WB 0x52B4UL + +#define MAX_NUM_ENGINES (XDMA_CHANNEL_NUM_MAX * 2) +#define H2C_CHANNEL_OFFSET 0x1000 +#define SGDMA_OFFSET_FROM_CHANNEL 0x4000 +#define CHANNEL_SPACING 0x100 +#define TARGET_SPACING 0x1000 + +#define BYPASS_MODE_SPACING 0x0100 + +/* obtain the 32 most significant (high) bits of a 32-bit or 64-bit address */ +#define PCI_DMA_H(addr) ((addr >> 16) >> 16) +/* obtain the 32 least significant (low) bits of a 32-bit or 64-bit address */ +#define PCI_DMA_L(addr) (addr & 0xffffffffUL) + +#ifndef VM_RESERVED + #define VMEM_FLAGS (VM_IO | VM_DONTEXPAND | VM_DONTDUMP) +#else + #define VMEM_FLAGS (VM_IO | VM_RESERVED) +#endif + +#ifdef __LIBXDMA_DEBUG__ +#define dbg_io pr_err +#define dbg_fops pr_err +#define dbg_perf pr_err +#define dbg_sg pr_err +#define dbg_tfr pr_err +#define dbg_irq pr_err +#define dbg_init pr_err +#define dbg_desc pr_err +#else +/* disable debugging */ +#define dbg_io(...) +#define dbg_fops(...) +#define dbg_perf(...) +#define dbg_sg(...) +#define dbg_tfr(...) +#define dbg_irq(...) +#define dbg_init(...) +#define dbg_desc(...) +#endif + +/* SECTION: Enum definitions */ +enum transfer_state { + TRANSFER_STATE_NEW = 0, + TRANSFER_STATE_SUBMITTED, + TRANSFER_STATE_COMPLETED, + TRANSFER_STATE_FAILED, + TRANSFER_STATE_ABORTED +}; + +enum shutdown_state { + ENGINE_SHUTDOWN_NONE = 0, /* No shutdown in progress */ + ENGINE_SHUTDOWN_REQUEST = 1, /* engine requested to shutdown */ + ENGINE_SHUTDOWN_IDLE = 2 /* engine has shutdown and is idle */ +}; + +enum dev_capabilities { + CAP_64BIT_DMA = 2, + CAP_64BIT_DESC = 4, + CAP_ENGINE_WRITE = 8, + CAP_ENGINE_READ = 16 +}; + +/* SECTION: Structure definitions */ + +struct config_regs { + u32 identifier; + u32 reserved_1[4]; + u32 msi_enable; +}; + +/** + * SG DMA Controller status and control registers + * + * These registers make the control interface for DMA transfers. + * + * It sits in End Point (FPGA) memory BAR[0] for 32-bit or BAR[0:1] for 64-bit. + * It references the first descriptor which exists in Root Complex (PC) memory. + * + * @note The registers must be accessed using 32-bit (PCI DWORD) read/writes, + * and their values are in little-endian byte ordering. + */ +struct engine_regs { + u32 identifier; + u32 control; + u32 control_w1s; + u32 control_w1c; + u32 reserved_1[12]; /* padding */ + + u32 status; + u32 status_rc; + u32 completed_desc_count; + u32 alignments; + u32 reserved_2[14]; /* padding */ + + u32 poll_mode_wb_lo; + u32 poll_mode_wb_hi; + u32 interrupt_enable_mask; + u32 interrupt_enable_mask_w1s; + u32 interrupt_enable_mask_w1c; + u32 reserved_3[9]; /* padding */ + + u32 perf_ctrl; + u32 perf_cyc_lo; + u32 perf_cyc_hi; + u32 perf_dat_lo; + u32 perf_dat_hi; + u32 perf_pnd_lo; + u32 perf_pnd_hi; +} __packed; + +struct engine_sgdma_regs { + u32 identifier; + u32 reserved_1[31]; /* padding */ + + /* bus address to first descriptor in Root Complex Memory */ + u32 first_desc_lo; + u32 first_desc_hi; + /* number of adjacent descriptors at first_desc */ + u32 first_desc_adjacent; + u32 credits; +} __packed; + +struct msix_vec_table_entry { + u32 msi_vec_addr_lo; + u32 msi_vec_addr_hi; + u32 msi_vec_data_lo; + u32 msi_vec_data_hi; +} __packed; + +struct msix_vec_table { + struct msix_vec_table_entry entry_list[32]; +} __packed; + +struct interrupt_regs { + u32 identifier; + u32 user_int_enable; + u32 user_int_enable_w1s; + u32 user_int_enable_w1c; + u32 channel_int_enable; + u32 channel_int_enable_w1s; + u32 channel_int_enable_w1c; + u32 reserved_1[9]; /* padding */ + + u32 user_int_request; + u32 channel_int_request; + u32 user_int_pending; + u32 channel_int_pending; + u32 reserved_2[12]; /* padding */ + + u32 user_msi_vector[8]; + u32 channel_msi_vector[8]; +} __packed; + +struct sgdma_common_regs { + u32 padding[8]; + u32 credit_mode_enable; + u32 credit_mode_enable_w1s; + u32 credit_mode_enable_w1c; +} __packed; + + +/* Structure for polled mode descriptor writeback */ +struct xdma_poll_wb { + u32 completed_desc_count; + u32 reserved_1[7]; +} __packed; + + +/** + * Descriptor for a single contiguous memory block transfer. + * + * Multiple descriptors are linked by means of the next pointer. An additional + * extra adjacent number gives the amount of extra contiguous descriptors. + * + * The descriptors are in root complex memory, and the bytes in the 32-bit + * words must be in little-endian byte ordering. + */ +struct xdma_desc { + u32 control; + u32 bytes; /* transfer length in bytes */ + u32 src_addr_lo; /* source address (low 32-bit) */ + u32 src_addr_hi; /* source address (high 32-bit) */ + u32 dst_addr_lo; /* destination address (low 32-bit) */ + u32 dst_addr_hi; /* destination address (high 32-bit) */ + /* + * next descriptor in the single-linked list of descriptors; + * this is the PCIe (bus) address of the next descriptor in the + * root complex memory + */ + u32 next_lo; /* next desc address (low 32-bit) */ + u32 next_hi; /* next desc address (high 32-bit) */ +} __packed; + +/* 32 bytes (four 32-bit words) or 64 bytes (eight 32-bit words) */ +struct xdma_result { + u32 status; + u32 length; + u32 reserved_1[6]; /* padding */ +} __packed; + +struct sw_desc { + dma_addr_t addr; + unsigned int len; +}; + +/* Describes a (SG DMA) single transfer for the engine */ +struct xdma_transfer { + struct list_head entry; /* queue of non-completed transfers */ + struct xdma_desc *desc_virt; /* virt addr of the 1st descriptor */ + dma_addr_t desc_bus; /* bus addr of the first descriptor */ + int desc_adjacent; /* adjacent descriptors at desc_bus */ + int desc_num; /* number of descriptors in transfer */ + enum dma_data_direction dir; + wait_queue_head_t wq; /* wait queue for transfer completion */ + + enum transfer_state state; /* state of the transfer */ + unsigned int flags; +#define XFER_FLAG_NEED_UNMAP 0x1 + int cyclic; /* flag if transfer is cyclic */ + int last_in_request; /* flag if last within request */ + unsigned int len; + struct sg_table *sgt; +}; + +struct xdma_request_cb { + struct sg_table *sgt; + unsigned int total_len; + u64 ep_addr; + + struct xdma_transfer xfer; + + unsigned int sw_desc_idx; + unsigned int sw_desc_cnt; + struct sw_desc sdesc[0]; +}; + +struct xdma_engine { + unsigned long magic; /* structure ID for sanity checks */ + struct xdma_dev *xdev; /* parent device */ + char name[5]; /* name of this engine */ + int version; /* version of this engine */ + //dev_t cdevno; /* character device major:minor */ + //struct cdev cdev; /* character device (embedded struct) */ + + /* HW register address offsets */ + struct engine_regs *regs; /* Control reg BAR offset */ + struct engine_sgdma_regs *sgdma_regs; /* SGDAM reg BAR offset */ + u32 bypass_offset; /* Bypass mode BAR offset */ + + /* Engine state, configuration and flags */ + enum shutdown_state shutdown; /* engine shutdown mode */ + enum dma_data_direction dir; + int device_open; /* flag if engine node open, ST mode only */ + int running; /* flag if the driver started engine */ + int non_incr_addr; /* flag if non-incremental addressing used */ + int streaming; + int addr_align; /* source/dest alignment in bytes */ + int len_granularity; /* transfer length multiple */ + int addr_bits; /* HW datapath address width */ + int channel; /* engine indices */ + int max_extra_adj; /* descriptor prefetch capability */ + int desc_dequeued; /* num descriptors of completed transfers */ + u32 status; /* last known status of device */ + u32 interrupt_enable_mask_value;/* only used for MSIX mode to store per-engine interrupt mask value */ + + /* Transfer list management */ + struct list_head transfer_list; /* queue of transfers */ + + /* Members applicable to AXI-ST C2H (cyclic) transfers */ + struct xdma_result *cyclic_result; + dma_addr_t cyclic_result_bus; /* bus addr for transfer */ + struct xdma_request_cb *cyclic_req; + struct sg_table cyclic_sgt; + u8 eop_found; /* used only for cyclic(rx:c2h) */ + + int rx_tail; /* follows the HW */ + int rx_head; /* where the SW reads from */ + int rx_overrun; /* flag if overrun occured */ + + /* for copy from cyclic buffer to user buffer */ + unsigned int user_buffer_index; + + /* Members associated with polled mode support */ + u8 *poll_mode_addr_virt; /* virt addr for descriptor writeback */ + dma_addr_t poll_mode_bus; /* bus addr for descriptor writeback */ + + /* Members associated with interrupt mode support */ + wait_queue_head_t shutdown_wq; /* wait queue for shutdown sync */ + spinlock_t lock; /* protects concurrent access */ + int prev_cpu; /* remember CPU# of (last) locker */ + int msix_irq_line; /* MSI-X vector for this engine */ + u32 irq_bitmask; /* IRQ bit mask for this engine */ + struct work_struct work; /* Work queue for interrupt handling */ + + spinlock_t desc_lock; /* protects concurrent access */ + dma_addr_t desc_bus; + struct xdma_desc *desc; + + /* for performance test support */ + struct xdma_performance_ioctl *xdma_perf; /* perf test control */ + wait_queue_head_t xdma_perf_wq; /* Perf test sync */ +}; + +struct xdma_user_irq { + struct xdma_dev *xdev; /* parent device */ + u8 user_idx; /* 0 ~ 15 */ + u8 events_irq; /* accumulated IRQs */ + spinlock_t events_lock; /* lock to safely update events_irq */ + wait_queue_head_t events_wq; /* wait queue to sync waiting threads */ + irq_handler_t handler; + + void *dev; +}; + +/* XDMA PCIe device specific book-keeping */ +#define XDEV_FLAG_OFFLINE 0x1 +struct xdma_dev { + struct list_head list_head; + struct list_head rcu_node; + + unsigned long magic; /* structure ID for sanity checks */ + struct pci_dev *pdev; /* pci device struct from probe() */ + int idx; /* dev index */ + + const char *mod_name; /* name of module owning the dev */ + + spinlock_t lock; /* protects concurrent access */ + unsigned int flags; + + /* PCIe BAR management */ + void *__iomem bar[XDMA_BAR_NUM]; /* addresses for mapped BARs */ + int user_bar_idx; /* BAR index of user logic */ + int config_bar_idx; /* BAR index of XDMA config logic */ + int bypass_bar_idx; /* BAR index of XDMA bypass logic */ + int regions_in_use; /* flag if dev was in use during probe() */ + int got_regions; /* flag if probe() obtained the regions */ + + int user_max; + int c2h_channel_max; + int h2c_channel_max; + + /* Interrupt management */ + int irq_count; /* interrupt counter */ + int irq_line; /* flag if irq allocated successfully */ + int msi_enabled; /* flag if msi was enabled for the device */ + int msix_enabled; /* flag if msi-x was enabled for the device */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,12,0) + struct msix_entry entry[32]; /* msi-x vector/entry table */ +#endif + struct xdma_user_irq user_irq[16]; /* user IRQ management */ + unsigned int mask_irq_user; + + /* XDMA engine management */ + int engines_num; /* Total engine count */ + u32 mask_irq_h2c; + u32 mask_irq_c2h; + struct xdma_engine engine_h2c[XDMA_CHANNEL_NUM_MAX]; + struct xdma_engine engine_c2h[XDMA_CHANNEL_NUM_MAX]; + + /* SD_Accel specific */ + enum dev_capabilities capabilities; + u64 feature_id; +}; + +static inline int xdma_device_flag_check(struct xdma_dev *xdev, unsigned int f) +{ + unsigned long flags; + + spin_lock_irqsave(&xdev->lock, flags); + if (xdev->flags & f) { + spin_unlock_irqrestore(&xdev->lock, flags); + return 1; + } + spin_unlock_irqrestore(&xdev->lock, flags); + return 0; +} + +static inline int xdma_device_flag_test_n_set(struct xdma_dev *xdev, + unsigned int f) +{ + unsigned long flags; + int rv = 0; + + spin_lock_irqsave(&xdev->lock, flags); + if (xdev->flags & f) { + spin_unlock_irqrestore(&xdev->lock, flags); + rv = 1; + } else + xdev->flags |= f; + spin_unlock_irqrestore(&xdev->lock, flags); + return rv; +} + +static inline void xdma_device_flag_set(struct xdma_dev *xdev, unsigned int f) +{ + unsigned long flags; + + spin_lock_irqsave(&xdev->lock, flags); + xdev->flags |= f; + spin_unlock_irqrestore(&xdev->lock, flags); +} + +static inline void xdma_device_flag_clear(struct xdma_dev *xdev, unsigned int f) +{ + unsigned long flags; + + spin_lock_irqsave(&xdev->lock, flags); + xdev->flags &= ~f; + spin_unlock_irqrestore(&xdev->lock, flags); +} + +void write_register(u32 value, void *iomem); +u32 read_register(void *iomem); + +struct xdma_dev *xdev_find_by_pdev(struct pci_dev *pdev); + +void xdma_device_offline(struct pci_dev *pdev, void *dev_handle); +void xdma_device_online(struct pci_dev *pdev, void *dev_handle); + +int xdma_performance_submit(struct xdma_dev *xdev, struct xdma_engine *engine); +struct xdma_transfer *engine_cyclic_stop(struct xdma_engine *engine); +void enable_perf(struct xdma_engine *engine); +void get_perf_stats(struct xdma_engine *engine); + +int xdma_cyclic_transfer_setup(struct xdma_engine *engine); +int xdma_cyclic_transfer_teardown(struct xdma_engine *engine); +ssize_t xdma_engine_read_cyclic(struct xdma_engine *, char __user *, size_t, + int); +int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg); + +#endif /* XDMA_LIB_H */ diff --git a/sources/xdma_driver/xdma/libxdma.o.ur-safe b/sources/xdma_driver/xdma/libxdma.o.ur-safe new file mode 100644 index 0000000..699074b --- /dev/null +++ b/sources/xdma_driver/xdma/libxdma.o.ur-safe @@ -0,0 +1,10 @@ +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-29fb +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-2a64 +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-2ac5 +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-2df4 +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-39f5 +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-3f12 +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-4ee8 +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-511e +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-5348 +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-61ea diff --git a/sources/xdma_driver/xdma/version.h b/sources/xdma_driver/xdma/version.h new file mode 100644 index 0000000..edd3abf --- /dev/null +++ b/sources/xdma_driver/xdma/version.h @@ -0,0 +1,28 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef __XDMA_VERSION_H__ +#define __XDMA_VERSION_H__ + +#define DRV_MOD_MAJOR 2017 +#define DRV_MOD_MINOR 1 +#define DRV_MOD_PATCHLEVEL 47 + +#define DRV_MODULE_VERSION \ + __stringify(DRV_MOD_MAJOR) "." \ + __stringify(DRV_MOD_MINOR) "." \ + __stringify(DRV_MOD_PATCHLEVEL) + +#define DRV_MOD_VERSION_NUMBER \ + ((DRV_MOD_MAJOR)*1000 + (DRV_MOD_MINOR)*100 + DRV_MOD_PATCHLEVEL) + +#endif /* ifndef __XDMA_VERSION_H__ */ diff --git a/sources/xdma_driver/xdma/xdma_cdev.c b/sources/xdma_driver/xdma/xdma_cdev.c new file mode 100644 index 0000000..b7981f6 --- /dev/null +++ b/sources/xdma_driver/xdma/xdma_cdev.c @@ -0,0 +1,539 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include "xdma_cdev.h" + +struct class *g_xdma_class; + +enum cdev_type { + CHAR_USER, + CHAR_CTRL, + CHAR_XVC, + CHAR_EVENTS, + CHAR_XDMA_H2C, + CHAR_XDMA_C2H, + CHAR_BYPASS_H2C, + CHAR_BYPASS_C2H, + CHAR_BYPASS, +}; + +static const char * const devnode_names[] = { + XDMA_NODE_NAME "%d_user", + XDMA_NODE_NAME "%d_control", + XDMA_NODE_NAME "%d_xvc", + XDMA_NODE_NAME "%d_events_%d", + XDMA_NODE_NAME "%d_h2c_%d", + XDMA_NODE_NAME "%d_c2h_%d", + XDMA_NODE_NAME "%d_bypass_h2c_%d", + XDMA_NODE_NAME "%d_bypass_c2h_%d", + XDMA_NODE_NAME "%d_bypass", +}; + +enum xpdev_flags_bits { + XDF_CDEV_USER, + XDF_CDEV_CTRL, + XDF_CDEV_XVC, + XDF_CDEV_EVENT, + XDF_CDEV_SG, + XDF_CDEV_BYPASS, +}; + +static inline void xpdev_flag_set(struct xdma_pci_dev *xpdev, + enum xpdev_flags_bits fbit) +{ + xpdev->flags |= 1 << fbit; +} + +static inline void xcdev_flag_clear(struct xdma_pci_dev *xpdev, + enum xpdev_flags_bits fbit) +{ + xpdev->flags &= ~(1 << fbit); +} + +static inline int xpdev_flag_test(struct xdma_pci_dev *xpdev, + enum xpdev_flags_bits fbit) +{ + return xpdev->flags & (1 << fbit); +} + +#ifdef __XDMA_SYSFS__ +ssize_t show_device_numbers(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct xdma_pci_dev *xpdev = (struct xdma_pci_dev *)dev_get_drvdata(dev); + + return snprintf(buf, PAGE_SIZE, "%d\t%d\n", + xpdev->major, xpdev->xdev->idx); +} + +static DEVICE_ATTR(xdma_dev_instance, S_IRUGO, show_device_numbers, NULL); +#endif + +static int config_kobject(struct xdma_cdev *xcdev, enum cdev_type type) +{ + int rv = -EINVAL; + struct xdma_dev *xdev = xcdev->xdev; + struct xdma_engine *engine = xcdev->engine; + + switch (type) { + case CHAR_XDMA_H2C: + case CHAR_XDMA_C2H: + case CHAR_BYPASS_H2C: + case CHAR_BYPASS_C2H: + BUG_ON(!engine); + rv = kobject_set_name(&xcdev->cdev.kobj, devnode_names[type], + xdev->idx, engine->channel); + break; + case CHAR_BYPASS: + case CHAR_USER: + case CHAR_CTRL: + case CHAR_XVC: + rv = kobject_set_name(&xcdev->cdev.kobj, devnode_names[type], + xdev->idx); + break; + case CHAR_EVENTS: + rv = kobject_set_name(&xcdev->cdev.kobj, devnode_names[type], + xdev->idx, xcdev->bar); + break; + default: + pr_warn("%s: UNKNOWN type 0x%x.\n", __func__, type); + break; + } + + if (rv) + pr_err("%s: type 0x%x, failed %d.\n", __func__, type, rv); + return rv; +} + +int xcdev_check(const char *fname, struct xdma_cdev *xcdev, bool check_engine) +{ + struct xdma_dev *xdev; + + if (!xcdev || xcdev->magic != MAGIC_CHAR) { + pr_info("%s, xcdev 0x%p, magic 0x%lx.\n", + fname, xcdev, xcdev ? xcdev->magic : 0xFFFFFFFF); + return -EINVAL; + } + + xdev = xcdev->xdev; + if (!xdev || xdev->magic != MAGIC_DEVICE) { + pr_info("%s, xdev 0x%p, magic 0x%lx.\n", + fname, xdev, xdev ? xdev->magic : 0xFFFFFFFF); + return -EINVAL; + } + + if (check_engine) { + struct xdma_engine *engine = xcdev->engine; + if (!engine || engine->magic != MAGIC_ENGINE) { + pr_info("%s, engine 0x%p, magic 0x%lx.\n", fname, + engine, engine ? engine->magic : 0xFFFFFFFF); + return -EINVAL; + } + } + + return 0; +} + +int char_open(struct inode *inode, struct file *file) +{ + struct xdma_cdev *xcdev; + + /* pointer to containing structure of the character device inode */ + xcdev = container_of(inode->i_cdev, struct xdma_cdev, cdev); + BUG_ON(xcdev->magic != MAGIC_CHAR); + /* create a reference to our char device in the opened file */ + file->private_data = xcdev; + + return 0; +} + +/* + * Called when the device goes from used to unused. + */ +int char_close(struct inode *inode, struct file *file) +{ + struct xdma_dev *xdev; + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + + BUG_ON(!xcdev); + BUG_ON(xcdev->magic != MAGIC_CHAR); + + /* fetch device specific data stored earlier during open */ + xdev = xcdev->xdev; + BUG_ON(!xdev); + BUG_ON(xdev->magic != MAGIC_DEVICE); + + return 0; +} + +/* create_xcdev() -- create a character device interface to data or control bus + * + * If at least one SG DMA engine is specified, the character device interface + * is coupled to the SG DMA file operations which operate on the data bus. If + * no engines are specified, the interface is coupled with the control bus. + */ + +static int create_sys_device(struct xdma_cdev *xcdev, enum cdev_type type) +{ + struct xdma_dev *xdev = xcdev->xdev; + struct xdma_engine *engine = xcdev->engine; + int last_param; + + if (type == CHAR_EVENTS) + last_param = xcdev->bar; + else + last_param = engine ? engine->channel : 0; + + xcdev->sys_device = device_create(g_xdma_class, &xdev->pdev->dev, + xcdev->cdevno, NULL, devnode_names[type], xdev->idx, + last_param); + + if (!xcdev->sys_device) { + pr_err("device_create(%s) failed\n", devnode_names[type]); + return -1; + } + + return 0; +} + +static int destroy_xcdev(struct xdma_cdev *cdev) +{ + if (!cdev) { + pr_warn("cdev NULL.\n"); + return 0; + } + if (cdev->magic != MAGIC_CHAR) { + pr_warn("cdev 0x%p magic mismatch 0x%lx\n", cdev, cdev->magic); + return 0; + } + BUG_ON(!cdev->xdev); + BUG_ON(!g_xdma_class); + BUG_ON(!cdev->sys_device); + + if (cdev->sys_device) + device_destroy(g_xdma_class, cdev->cdevno); + + cdev_del(&cdev->cdev); + + return 0; +} + +static int create_xcdev(struct xdma_pci_dev *xpdev, struct xdma_cdev *xcdev, + int bar, struct xdma_engine *engine, + enum cdev_type type) +{ + int rv; + int minor; + struct xdma_dev *xdev = xpdev->xdev; + dev_t dev; + + spin_lock_init(&xcdev->lock); + /* new instance? */ + if (!xpdev->major) { + /* allocate a dynamically allocated char device node */ + int rv = alloc_chrdev_region(&dev, XDMA_MINOR_BASE, + XDMA_MINOR_COUNT, XDMA_NODE_NAME); + + if (rv) { + pr_err("unable to allocate cdev region %d.\n", rv); + return rv; + } + xpdev->major = MAJOR(dev); + } + + /* + * do not register yet, create kobjects and name them, + */ + xcdev->magic = MAGIC_CHAR; + xcdev->cdev.owner = THIS_MODULE; + xcdev->xpdev = xpdev; + xcdev->xdev = xdev; + xcdev->engine = engine; + xcdev->bar = bar; + + rv = config_kobject(xcdev, type); + if (rv < 0) + return rv; + + switch (type) { + case CHAR_USER: + case CHAR_CTRL: + /* minor number is type index for non-SGDMA interfaces */ + minor = type; + cdev_ctrl_init(xcdev); + break; + case CHAR_XVC: + /* minor number is type index for non-SGDMA interfaces */ + minor = type; + cdev_xvc_init(xcdev); + break; + case CHAR_XDMA_H2C: + minor = 32 + engine->channel; + cdev_sgdma_init(xcdev); + break; + case CHAR_XDMA_C2H: + minor = 36 + engine->channel; + cdev_sgdma_init(xcdev); + break; + case CHAR_EVENTS: + minor = 10 + bar; + cdev_event_init(xcdev); + break; + case CHAR_BYPASS_H2C: + minor = 64 + engine->channel; + cdev_bypass_init(xcdev); + break; + case CHAR_BYPASS_C2H: + minor = 68 + engine->channel; + cdev_bypass_init(xcdev); + break; + case CHAR_BYPASS: + minor = 100; + cdev_bypass_init(xcdev); + break; + default: + pr_info("type 0x%x NOT supported.\n", type); + return -EINVAL; + } + xcdev->cdevno = MKDEV(xpdev->major, minor); + + /* bring character device live */ + rv = cdev_add(&xcdev->cdev, xcdev->cdevno, 1); + if (rv < 0) { + pr_err("cdev_add failed %d, type 0x%x.\n", rv, type); + goto unregister_region; + } + + dbg_init("xcdev 0x%p, %u:%u, %s, type 0x%x.\n", + xcdev, xpdev->major, minor, xcdev->cdev.kobj.name, type); + + /* create device on our class */ + if (g_xdma_class) { + rv = create_sys_device(xcdev, type); + if (rv < 0) + goto del_cdev; + } + + return 0; + +del_cdev: + cdev_del(&xcdev->cdev); +unregister_region: + unregister_chrdev_region(dev, XDMA_MINOR_COUNT); + return rv; +} + +void xpdev_destroy_interfaces(struct xdma_pci_dev *xpdev) +{ + int i; + +#ifdef __XDMA_SYSFS__ + device_remove_file(&xpdev->pdev->dev, &dev_attr_xdma_dev_instance); +#endif + + if (xpdev_flag_test(xpdev, XDF_CDEV_SG)) { + /* iterate over channels */ + for (i = 0; i < xpdev->h2c_channel_max; i++) + /* remove SG DMA character device */ + destroy_xcdev(&xpdev->sgdma_h2c_cdev[i]); + for (i = 0; i < xpdev->c2h_channel_max; i++) + destroy_xcdev(&xpdev->sgdma_c2h_cdev[i]); + } + + if (xpdev_flag_test(xpdev, XDF_CDEV_EVENT)) { + for (i = 0; i < xpdev->user_max; i++) + destroy_xcdev(&xpdev->events_cdev[i]); + } + + /* remove control character device */ + if (xpdev_flag_test(xpdev, XDF_CDEV_CTRL)) { + destroy_xcdev(&xpdev->ctrl_cdev); + } + + /* remove user character device */ + if (xpdev_flag_test(xpdev, XDF_CDEV_USER)) { + destroy_xcdev(&xpdev->user_cdev); + } + + if (xpdev_flag_test(xpdev, XDF_CDEV_XVC)) { + destroy_xcdev(&xpdev->xvc_cdev); + } + + if (xpdev_flag_test(xpdev, XDF_CDEV_BYPASS)) { + /* iterate over channels */ + for (i = 0; i < xpdev->h2c_channel_max; i++) + /* remove DMA Bypass character device */ + destroy_xcdev(&xpdev->bypass_h2c_cdev[i]); + for (i = 0; i < xpdev->c2h_channel_max; i++) + destroy_xcdev(&xpdev->bypass_c2h_cdev[i]); + destroy_xcdev(&xpdev->bypass_cdev_base); + } + + if (xpdev->major) + unregister_chrdev_region(MKDEV(xpdev->major, XDMA_MINOR_BASE), XDMA_MINOR_COUNT); +} + +int xpdev_create_interfaces(struct xdma_pci_dev *xpdev) +{ + struct xdma_dev *xdev = xpdev->xdev; + struct xdma_engine *engine; + int i; + int rv = 0; + + /* initialize control character device */ + rv = create_xcdev(xpdev, &xpdev->ctrl_cdev, xdev->config_bar_idx, + NULL, CHAR_CTRL); + if (rv < 0) { + pr_err("create_char(ctrl_cdev) failed\n"); + goto fail; + } + xpdev_flag_set(xpdev, XDF_CDEV_CTRL); + + /* initialize events character device */ + for (i = 0; i < xpdev->user_max; i++) { + rv = create_xcdev(xpdev, &xpdev->events_cdev[i], i, NULL, + CHAR_EVENTS); + if (rv < 0) { + pr_err("create char event %d failed, %d.\n", i, rv); + goto fail; + } + } + xpdev_flag_set(xpdev, XDF_CDEV_EVENT); + + /* iterate over channels */ + for (i = 0; i < xpdev->h2c_channel_max; i++) { + engine = &xdev->engine_h2c[i]; + + if (engine->magic != MAGIC_ENGINE) + continue; + + rv = create_xcdev(xpdev, &xpdev->sgdma_h2c_cdev[i], i, engine, + CHAR_XDMA_H2C); + if (rv < 0) { + pr_err("create char h2c %d failed, %d.\n", i, rv); + goto fail; + } + } + + for (i = 0; i < xpdev->c2h_channel_max; i++) { + engine = &xdev->engine_c2h[i]; + + if (engine->magic != MAGIC_ENGINE) + continue; + + rv = create_xcdev(xpdev, &xpdev->sgdma_c2h_cdev[i], i, engine, + CHAR_XDMA_C2H); + if (rv < 0) { + pr_err("create char c2h %d failed, %d.\n", i, rv); + goto fail; + } + } + xpdev_flag_set(xpdev, XDF_CDEV_SG); + + /* ??? Bypass */ + /* Initialize Bypass Character Device */ + if (xdev->bypass_bar_idx > 0){ + for (i = 0; i < xpdev->h2c_channel_max; i++) { + engine = &xdev->engine_h2c[i]; + + if (engine->magic != MAGIC_ENGINE) + continue; + + rv = create_xcdev(xpdev, &xpdev->bypass_h2c_cdev[i], i, + engine, CHAR_BYPASS_H2C); + if (rv < 0) { + pr_err("create h2c %d bypass I/F failed, %d.\n", + i, rv); + goto fail; + } + } + + for (i = 0; i < xpdev->c2h_channel_max; i++) { + engine = &xdev->engine_c2h[i]; + + if (engine->magic != MAGIC_ENGINE) + continue; + + rv = create_xcdev(xpdev, &xpdev->bypass_c2h_cdev[i], i, + engine, CHAR_BYPASS_C2H); + if (rv < 0) { + pr_err("create c2h %d bypass I/F failed, %d.\n", + i, rv); + goto fail; + } + } + + rv = create_xcdev(xpdev, &xpdev->bypass_cdev_base, + xdev->bypass_bar_idx, NULL, CHAR_BYPASS); + if (rv < 0) { + pr_err("create bypass failed %d.\n", rv); + goto fail; + } + xpdev_flag_set(xpdev, XDF_CDEV_BYPASS); + } + + /* initialize user character device */ + if (xdev->user_bar_idx >= 0) { + rv = create_xcdev(xpdev, &xpdev->user_cdev, xdev->user_bar_idx, + NULL, CHAR_USER); + if (rv < 0) { + pr_err("create_char(user_cdev) failed\n"); + goto fail; + } + xpdev_flag_set(xpdev, XDF_CDEV_USER); + + /* xvc */ + rv = create_xcdev(xpdev, &xpdev->xvc_cdev, xdev->user_bar_idx, + NULL, CHAR_XVC); + if (rv < 0) { + pr_err("create xvc failed, %d.\n", rv); + goto fail; + } + xpdev_flag_set(xpdev, XDF_CDEV_XVC); + } + +#ifdef __XDMA_SYSFS__ + /* sys file */ + rv = device_create_file(&xpdev->pdev->dev, + &dev_attr_xdma_dev_instance); + if (rv) { + pr_err("Failed to create device file \n"); + goto fail; + } +#endif + + return 0; + +fail: + rv = -1; + xpdev_destroy_interfaces(xpdev); + return rv; +} + +int xdma_cdev_init(void) +{ + g_xdma_class = class_create(THIS_MODULE, XDMA_NODE_NAME); + if (IS_ERR(g_xdma_class)) { + dbg_init(XDMA_NODE_NAME ": failed to create class"); + return -1; + } + + return 0; +} + +void xdma_cdev_cleanup(void) +{ + if (g_xdma_class) + class_destroy(g_xdma_class); +} diff --git a/sources/xdma_driver/xdma/xdma_cdev.h b/sources/xdma_driver/xdma/xdma_cdev.h new file mode 100644 index 0000000..9570066 --- /dev/null +++ b/sources/xdma_driver/xdma/xdma_cdev.h @@ -0,0 +1,44 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef __XDMA_CHRDEV_H__ +#define __XDMA_CHRDEV_H__ + +#include <linux/kernel.h> +#include <linux/types.h> +#include <linux/uaccess.h> +#include <linux/errno.h> +#include "xdma_mod.h" + +#define XDMA_NODE_NAME "xdma" +#define XDMA_MINOR_BASE (0) +#define XDMA_MINOR_COUNT (255) + +void xdma_cdev_cleanup(void); +int xdma_cdev_init(void); + +int char_open(struct inode *inode, struct file *file); +int char_close(struct inode *inode, struct file *file); +int xcdev_check(const char *, struct xdma_cdev *, bool); + +void cdev_ctrl_init(struct xdma_cdev *xcdev); +void cdev_xvc_init(struct xdma_cdev *xcdev); +void cdev_event_init(struct xdma_cdev *xcdev); +void cdev_sgdma_init(struct xdma_cdev *xcdev); +void cdev_bypass_init(struct xdma_cdev *xcdev); + +void xpdev_destroy_interfaces(struct xdma_pci_dev *xpdev); +int xpdev_create_interfaces(struct xdma_pci_dev *xpdev); + +int bridge_mmap(struct file *file, struct vm_area_struct *vma); + +#endif /* __XDMA_CHRDEV_H__ */ diff --git a/sources/xdma_driver/xdma/xdma_mod.c b/sources/xdma_driver/xdma/xdma_mod.c new file mode 100644 index 0000000..d6f724a --- /dev/null +++ b/sources/xdma_driver/xdma/xdma_mod.c @@ -0,0 +1,347 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include <linux/ioctl.h> +#include <linux/types.h> +#include <linux/errno.h> +#include <linux/aer.h> +/* include early, to verify it depends only on the headers above */ +#include "libxdma_api.h" +#include "libxdma.h" +#include "xdma_mod.h" +#include "xdma_cdev.h" +#include "version.h" + +#define DRV_MODULE_NAME "xdma" +#define DRV_MODULE_DESC "Xilinx XDMA Reference Driver" +#define DRV_MODULE_RELDATE "Feb. 2018" + +static char version[] = + DRV_MODULE_DESC " " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n"; + +MODULE_AUTHOR("Xilinx, Inc."); +MODULE_DESCRIPTION(DRV_MODULE_DESC); +MODULE_VERSION(DRV_MODULE_VERSION); +MODULE_LICENSE("Dual BSD/GPL"); + +/* SECTION: Module global variables */ +static int xpdev_cnt = 0; + +static const struct pci_device_id pci_ids[] = { + { PCI_DEVICE(0x10ee, 0x903f), }, + { PCI_DEVICE(0x10ee, 0x9038), }, + { PCI_DEVICE(0x10ee, 0x9028), }, + { PCI_DEVICE(0x10ee, 0x9018), }, + { PCI_DEVICE(0x10ee, 0x9034), }, + { PCI_DEVICE(0x10ee, 0x9024), }, + { PCI_DEVICE(0x10ee, 0x9014), }, + { PCI_DEVICE(0x10ee, 0x9032), }, + { PCI_DEVICE(0x10ee, 0x9022), }, + { PCI_DEVICE(0x10ee, 0x9012), }, + { PCI_DEVICE(0x10ee, 0x9031), }, + { PCI_DEVICE(0x10ee, 0x9021), }, + { PCI_DEVICE(0x10ee, 0x9011), }, + + { PCI_DEVICE(0x10ee, 0x8011), }, + { PCI_DEVICE(0x10ee, 0x8012), }, + { PCI_DEVICE(0x10ee, 0x8014), }, + { PCI_DEVICE(0x10ee, 0x8018), }, + { PCI_DEVICE(0x10ee, 0x8021), }, + { PCI_DEVICE(0x10ee, 0x8022), }, + { PCI_DEVICE(0x10ee, 0x8024), }, + { PCI_DEVICE(0x10ee, 0x8028), }, + { PCI_DEVICE(0x10ee, 0x8031), }, + { PCI_DEVICE(0x10ee, 0x8032), }, + { PCI_DEVICE(0x10ee, 0x8034), }, + { PCI_DEVICE(0x10ee, 0x8038), }, + + { PCI_DEVICE(0x10ee, 0x7011), }, + { PCI_DEVICE(0x10ee, 0x7012), }, + { PCI_DEVICE(0x10ee, 0x7014), }, + { PCI_DEVICE(0x10ee, 0x7018), }, + { PCI_DEVICE(0x10ee, 0x7021), }, + { PCI_DEVICE(0x10ee, 0x7022), }, + { PCI_DEVICE(0x10ee, 0x7024), }, + { PCI_DEVICE(0x10ee, 0x7028), }, + { PCI_DEVICE(0x10ee, 0x7031), }, + { PCI_DEVICE(0x10ee, 0x7032), }, + { PCI_DEVICE(0x10ee, 0x7034), }, + { PCI_DEVICE(0x10ee, 0x7038), }, + + { PCI_DEVICE(0x10ee, 0x6828), }, + { PCI_DEVICE(0x10ee, 0x6830), }, + { PCI_DEVICE(0x10ee, 0x6928), }, + { PCI_DEVICE(0x10ee, 0x6930), }, + { PCI_DEVICE(0x10ee, 0x6A28), }, + { PCI_DEVICE(0x10ee, 0x6A30), }, + { PCI_DEVICE(0x10ee, 0x6D30), }, + + { PCI_DEVICE(0x10ee, 0x4808), }, + { PCI_DEVICE(0x10ee, 0x4828), }, + { PCI_DEVICE(0x10ee, 0x4908), }, + { PCI_DEVICE(0x10ee, 0x4A28), }, + { PCI_DEVICE(0x10ee, 0x4B28), }, + + { PCI_DEVICE(0x10ee, 0x2808), }, + +#ifdef INTERNAL_TESTING + { PCI_DEVICE(0x1d0f, 0x1042), 0}, +#endif + {0,} +}; +MODULE_DEVICE_TABLE(pci, pci_ids); + +static void xpdev_free(struct xdma_pci_dev *xpdev) +{ + struct xdma_dev *xdev = xpdev->xdev; + + pr_info("xpdev 0x%p, destroy_interfaces, xdev 0x%p.\n", xpdev, xdev); + xpdev_destroy_interfaces(xpdev); + xpdev->xdev = NULL; + pr_info("xpdev 0x%p, xdev 0x%p xdma_device_close.\n", xpdev, xdev); + xdma_device_close(xpdev->pdev, xdev); + xpdev_cnt--; + + kfree(xpdev); +} + +static struct xdma_pci_dev *xpdev_alloc(struct pci_dev *pdev) +{ + struct xdma_pci_dev *xpdev = kmalloc(sizeof(*xpdev), GFP_KERNEL); + + if (!xpdev) + return NULL; + memset(xpdev, 0, sizeof(*xpdev)); + + xpdev->magic = MAGIC_DEVICE; + xpdev->pdev = pdev; + xpdev->user_max = MAX_USER_IRQ; + xpdev->h2c_channel_max = XDMA_CHANNEL_NUM_MAX; + xpdev->c2h_channel_max = XDMA_CHANNEL_NUM_MAX; + + xpdev_cnt++; + return xpdev; +} + +static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id) +{ + int rv = 0; + struct xdma_pci_dev *xpdev = NULL; + struct xdma_dev *xdev; + void *hndl; + + xpdev = xpdev_alloc(pdev); + if (!xpdev) + return -ENOMEM; + + hndl = xdma_device_open(DRV_MODULE_NAME, pdev, &xpdev->user_max, + &xpdev->h2c_channel_max, &xpdev->c2h_channel_max); + if (!hndl) + return -EINVAL; + + BUG_ON(xpdev->user_max > MAX_USER_IRQ); + BUG_ON(xpdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX); + BUG_ON(xpdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX); + + if (!xpdev->h2c_channel_max && !xpdev->c2h_channel_max) + pr_warn("NO engine found!\n"); + + if (xpdev->user_max) { + u32 mask = (1 << (xpdev->user_max + 1)) - 1; + + rv = xdma_user_isr_enable(hndl, mask); + if (rv) + goto err_out; + } + + /* make sure no duplicate */ + xdev = xdev_find_by_pdev(pdev); + if (!xdev) { + pr_warn("NO xdev found!\n"); + return -EINVAL; + } + BUG_ON(hndl != xdev ); + + pr_info("%s xdma%d, pdev 0x%p, xdev 0x%p, 0x%p, usr %d, ch %d,%d.\n", + dev_name(&pdev->dev), xdev->idx, pdev, xpdev, xdev, + xpdev->user_max, xpdev->h2c_channel_max, + xpdev->c2h_channel_max); + + xpdev->xdev = hndl; + + rv = xpdev_create_interfaces(xpdev); + if (rv) + goto err_out; + + dev_set_drvdata(&pdev->dev, xpdev); + + return 0; + +err_out: + pr_err("pdev 0x%p, err %d.\n", pdev, rv); + xpdev_free(xpdev); + return rv; +} + +static void remove_one(struct pci_dev *pdev) +{ + struct xdma_pci_dev *xpdev; + + if (!pdev) + return; + + xpdev = dev_get_drvdata(&pdev->dev); + if (!xpdev) + return; + + pr_info("pdev 0x%p, xdev 0x%p, 0x%p.\n", + pdev, xpdev, xpdev->xdev); + xpdev_free(xpdev); + + dev_set_drvdata(&pdev->dev, NULL); +} + +static pci_ers_result_t xdma_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev); + + switch (state) { + case pci_channel_io_normal: + return PCI_ERS_RESULT_CAN_RECOVER; + case pci_channel_io_frozen: + pr_warn("dev 0x%p,0x%p, frozen state error, reset controller\n", + pdev, xpdev); + xdma_device_offline(pdev, xpdev->xdev); + pci_disable_device(pdev); + return PCI_ERS_RESULT_NEED_RESET; + case pci_channel_io_perm_failure: + pr_warn("dev 0x%p,0x%p, failure state error, req. disconnect\n", + pdev, xpdev); + return PCI_ERS_RESULT_DISCONNECT; + } + return PCI_ERS_RESULT_NEED_RESET; +} + +static pci_ers_result_t xdma_slot_reset(struct pci_dev *pdev) +{ + struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev); + + pr_info("0x%p restart after slot reset\n", xpdev); + if (pci_enable_device_mem(pdev)) { + pr_info("0x%p failed to renable after slot reset\n", xpdev); + return PCI_ERS_RESULT_DISCONNECT; + } + + pci_set_master(pdev); + pci_restore_state(pdev); + pci_save_state(pdev); + xdma_device_online(pdev, xpdev->xdev); + + return PCI_ERS_RESULT_RECOVERED; +} + +static void xdma_error_resume(struct pci_dev *pdev) +{ + struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev); + + pr_info("dev 0x%p,0x%p.\n", pdev, xpdev); +#if KERNEL_VERSION(5, 7, 0) <= LINUX_VERSION_CODE + pci_aer_clear_nonfatal_status(pdev); +#else + pci_cleanup_aer_uncorrect_error_status(pdev); +#endif +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,13,0) +static void xdma_reset_prepare(struct pci_dev *pdev) +{ + struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev); + + pr_info("dev 0x%p,0x%p.\n", pdev, xpdev); + xdma_device_offline(pdev, xpdev->xdev); +} + +static void xdma_reset_done(struct pci_dev *pdev) +{ + struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev); + + pr_info("dev 0x%p,0x%p.\n", pdev, xpdev); + xdma_device_online(pdev, xpdev->xdev); +} + +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) +static void xdma_reset_notify(struct pci_dev *pdev, bool prepare) +{ + struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev); + + pr_info("dev 0x%p,0x%p, prepare %d.\n", pdev, xpdev, prepare); + + if (prepare) + xdma_device_offline(pdev, xpdev->xdev); + else + xdma_device_online(pdev, xpdev->xdev); +} +#endif + +static const struct pci_error_handlers xdma_err_handler = { + .error_detected = xdma_error_detected, + .slot_reset = xdma_slot_reset, + .resume = xdma_error_resume, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,13,0) + .reset_prepare = xdma_reset_prepare, + .reset_done = xdma_reset_done, +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) + .reset_notify = xdma_reset_notify, +#endif +}; + +static struct pci_driver pci_driver = { + .name = DRV_MODULE_NAME, + .id_table = pci_ids, + .probe = probe_one, + .remove = remove_one, + .err_handler = &xdma_err_handler, +}; + +static int __init xdma_mod_init(void) +{ + int rv; + extern unsigned int desc_blen_max; + extern unsigned int sgdma_timeout; + + pr_info("%s", version); + + if (desc_blen_max > XDMA_DESC_BLEN_MAX) + desc_blen_max = XDMA_DESC_BLEN_MAX; + pr_info("desc_blen_max: 0x%x/%u, sgdma_timeout: %u sec.\n", + desc_blen_max, desc_blen_max, sgdma_timeout); + + rv = xdma_cdev_init(); + if (rv < 0) + return rv; + + return pci_register_driver(&pci_driver); +} + +static void __exit xdma_mod_exit(void) +{ + /* unregister this driver from the PCI bus driver */ + dbg_init("pci_unregister_driver.\n"); + pci_unregister_driver(&pci_driver); + xdma_cdev_cleanup(); +} + +module_init(xdma_mod_init); +module_exit(xdma_mod_exit); diff --git a/sources/xdma_driver/xdma/xdma_mod.h b/sources/xdma_driver/xdma/xdma_mod.h new file mode 100644 index 0000000..93202b1 --- /dev/null +++ b/sources/xdma_driver/xdma/xdma_mod.h @@ -0,0 +1,98 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef __XDMA_MODULE_H__ +#define __XDMA_MODULE_H__ + +#include <linux/types.h> +#include <linux/module.h> +#include <linux/cdev.h> +#include <linux/dma-mapping.h> +#include <linux/delay.h> +#include <linux/fb.h> +#include <linux/fs.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/jiffies.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/mm_types.h> +#include <linux/poll.h> +#include <linux/pci.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/workqueue.h> +#include <linux/aio.h> +#include <linux/splice.h> +#include <linux/version.h> +#include <linux/uio.h> + +#include "libxdma.h" + +#define MAGIC_ENGINE 0xEEEEEEEEUL +#define MAGIC_DEVICE 0xDDDDDDDDUL +#define MAGIC_CHAR 0xCCCCCCCCUL +#define MAGIC_BITSTREAM 0xBBBBBBBBUL + +struct xdma_cdev { + unsigned long magic; /* structure ID for sanity checks */ + struct xdma_pci_dev *xpdev; + struct xdma_dev *xdev; + dev_t cdevno; /* character device major:minor */ + struct cdev cdev; /* character device embedded struct */ + int bar; /* PCIe BAR for HW access, if needed */ + unsigned long base; /* bar access offset */ + struct xdma_engine *engine; /* engine instance, if needed */ + struct xdma_user_irq *user_irq; /* IRQ value, if needed */ + struct device *sys_device; /* sysfs device */ + spinlock_t lock; +}; + +/* XDMA PCIe device specific book-keeping */ +struct xdma_pci_dev { + unsigned long magic; /* structure ID for sanity checks */ + struct pci_dev *pdev; /* pci device struct from probe() */ + struct xdma_dev *xdev; + int major; /* major number */ + int instance; /* instance number */ + int user_max; + int c2h_channel_max; + int h2c_channel_max; + + unsigned int flags; + /* character device structures */ + struct xdma_cdev ctrl_cdev; + struct xdma_cdev sgdma_c2h_cdev[XDMA_CHANNEL_NUM_MAX]; + struct xdma_cdev sgdma_h2c_cdev[XDMA_CHANNEL_NUM_MAX]; + struct xdma_cdev events_cdev[16]; + + struct xdma_cdev user_cdev; + struct xdma_cdev bypass_c2h_cdev[XDMA_CHANNEL_NUM_MAX]; + struct xdma_cdev bypass_h2c_cdev[XDMA_CHANNEL_NUM_MAX]; + struct xdma_cdev bypass_cdev_base; + + struct xdma_cdev xvc_cdev; + + void *data; +}; + +struct xdma_io_cb { + void __user *buf; + size_t len; + unsigned int pages_nr; + struct sg_table sgt; + struct page **pages; +}; + +#endif /* ifndef __XDMA_MODULE_H__ */ |