[PATCH] Staging : Add RIFFA PCIe driver

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



This patch adds RIFFA PCIe linux driver for https://github.com/promach/riffa/tree/full_duplex/driver/linux

TODO:

1) this driver needs further speed optimization although it can now achieve defined PCIe speed grade
2) solve all the coding style errors from scripts/checkpatch.pl

Signed-off-by: PHUNG CHENG FEI (feiphung@xxxxxxxxxxx)

---
Changes in v1:
  - added full-duplex capability

 drivers/staging/riffa/Makefile       |  157 +++
 drivers/staging/riffa/README.txt     |   38 +
 drivers/staging/riffa/circ_queue.c   |  188 +++
 drivers/staging/riffa/circ_queue.h   |   96 ++
 drivers/staging/riffa/riffa.c        |  152 +++
 drivers/staging/riffa/riffa.h        |  121 ++
 drivers/staging/riffa/riffa_driver.c | 1633 ++++++++++++++++++++++++++
 drivers/staging/riffa/riffa_driver.h |  131 +++
 8 files changed, 2516 insertions(+)
 create mode 100644 drivers/staging/riffa/Makefile
 create mode 100644 drivers/staging/riffa/README.txt
 create mode 100644 drivers/staging/riffa/circ_queue.c
 create mode 100644 drivers/staging/riffa/circ_queue.h
 create mode 100644 drivers/staging/riffa/riffa.c
 create mode 100644 drivers/staging/riffa/riffa.h
 create mode 100644 drivers/staging/riffa/riffa_driver.c
 create mode 100644 drivers/staging/riffa/riffa_driver.h

diff --git a/drivers/staging/riffa/Makefile b/drivers/staging/riffa/Makefile
new file mode 100644
index 000000000000..3e3cb0c4a387
--- /dev/null
+++ b/drivers/staging/riffa/Makefile
@@ -0,0 +1,157 @@
+# ----------------------------------------------------------------------
+# Copyright (c) 2016, The Regents of the University of California All
+# rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#
+#     * Neither the name of The Regents of the University of California
+#       nor the names of its contributors may be used to endorse or
+#       promote products derived from this software without specific
+#       prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL REGENTS OF THE
+# UNIVERSITY OF CALIFORNIA BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+# ----------------------------------------------------------------------
+
+# Filename: Makefile
+# Version: 2.0
+# Description: Makefile for Linux PCIe device driver for RIFFA.
+# Author: Matthew Jacobsen
+# History: @mattj: Initial release. Version 2.0.
+
+# You must specify the following variables. You can leave the defaults if you
+# like, but make sure they will work in your system.
+# The VENDOR_ID _must_ match what is configured on your FPGA's PCIe endpoint
+# header. Xilinx has a VENDOR_ID = 10EE.
+NAME := riffa
+VENDOR_ID0 := 10EE
+VENDOR_ID1 := 1172
+MAJNUM := 100
+
+# Build variables
+KVER := $(shell uname -r)
+KDIR := /lib/modules/`uname -r`/build
+RHR := /etc/redhat-release
+LIB_SRCS := riffa.c
+LIB_OBJS := $(patsubst %.c,%.o,$(LIB_SRCS))
+LIB_HDR := riffa.h
+LIB_VER_MAJ := 1
+LIB_VER_MIN := 0
+LIB_VER := $(LIB_VER_MAJ).$(LIB_VER_MIN)
+DRVR_HDR := riffa_driver.h
+DBUGVAL := DBUG
+
+obj-m += $(NAME).o
+$(NAME)-y := riffa_driver.o circ_queue.o
+
+# Helper functions
+define assert
+  $(if $1,,$(error Assertion failed: $2))
+endef
+define assert-not-null
+  $(call assert,$($1),The variable "$1" is null, please specify it.)
+endef
+define assert-variables
+       $(call assert-not-null,NAME)
+       $(call assert-not-null,MAJNUM)
+       $(call assert-not-null,VENDOR_ID0)
+       $(call assert-not-null,VENDOR_ID1)
+       @printf "Compiling driver for kernel: %s with the following values\n" $(KVER)
+       @printf "     NAME: '%s'\n" $(NAME)
+       @printf "   MAJNUM: '%s'\n" $(MAJNUM)
+       @printf "VENDOR_ID0: '%s'\n" $(VENDOR_ID0)
+       @printf "VENDOR_ID1: '%s'\n" $(VENDOR_ID1)
+       @printf "\n"
+endef
+
+all: builddvr
+debug: CC += -DDEBUG -g -pthread
+debug: DBUGVAL = DEBUG
+debug: builddvr
+builddvr: $(NAME).ko $(NAME).so.$(LIB_VER)
+
+$(NAME).ko: *.c *.h
+       $(call assert-variables)
+       sed -i 's/#define MAJOR_NUM [^\n]*/#define MAJOR_NUM $(MAJNUM)/g' $(DRVR_HDR)
+       sed -i 's/#define DEVICE_NAME [^\n]*/#define DEVICE_NAME "$(NAME)"/g' $(DRVR_HDR)
+       sed -i 's/#define VENDOR_ID0 [^\n]*/#define VENDOR_ID0 0x$(VENDOR_ID0)/g' $(DRVR_HDR)
+       sed -i 's/#define VENDOR_ID1 [^\n]*/#define VENDOR_ID1 0x$(VENDOR_ID1)/g' $(DRVR_HDR)
+       sed -i 's/#define DEBUG [^\n]*/#define DBUG 1/g' $(DRVR_HDR)
+       sed -i 's/#define DBUG [^\n]*/#define $(DBUGVAL) 1/g' $(DRVR_HDR)
+       make -C $(KDIR) SUBDIRS=`pwd` modules
+       rm -rf $(LIB_OBJS)
+
+$(NAME).so.$(LIB_VER): $(LIB_OBJS)
+       $(CC) -shared -Wl,-soname,lib$(NAME).so.$(LIB_VER_MAJ) -o lib$@ $^
+
+$(LIB_OBJS): $(LIB_SRCS)
+       $(CC) -g -Wall -fPIC -c $^
+
+load: $(NAME).ko
+       insmod $(NAME).ko
+
+unload:
+       rmmod $(NAME)
+
+clean:
+       rm -Rf *.ko *.cmd *.o *.so *.so.* .*.cmd Module.symvers Module.markers modules.order *.mod.c .tmp_versions
+
+setup:
+       if [ -f "$(RHR)" ]; then yum install kernel-devel-`uname -r`;\
+       else apt-get install linux-headers-`uname -r`; fi
+
+install: $(NAME).so.$(LIB_VER) $(NAME).ko
+       mkdir -p /lib/modules/$(KVER)/kernel/drivers/$(NAME)
+       cp $(NAME).ko /lib/modules/$(KVER)/kernel/drivers/$(NAME)/
+       if [ -f "$(RHR)" ]; then\
+               printf "%b\n" "#!/bin/sh\nexec /sbin/modprobe $(NAME) >/dev/null 2>&1" > /etc/sysconfig/modules/$(NAME).modules;\
+               chmod 755 /etc/sysconfig/modules/$(NAME).modules;\
+       else\
+               if ! grep -Fxq "$(NAME)" /etc/modules; then echo "$(NAME)" >> /etc/modules; fi;\
+       fi
+       printf "%b\n" "KERNEL==\"$(NAME)\", MODE=\"777\", GROUP=\"root\"" > /etc/udev/rules.d/99-$(NAME).rules
+       printf "/usr/local/lib\n" > $(NAME).conf
+       mv $(NAME).conf /etc/ld.so.conf.d/
+       cp $(DRVR_HDR) /usr/local/include/
+       cp $(LIB_HDR) /usr/local/include/
+       mv lib$(NAME).so.1.0 /usr/local/lib
+       ln -sf /usr/local/lib/lib$(NAME).so.$(LIB_VER) /usr/local/lib/lib$(NAME).so.$(LIB_VER_MAJ)
+       ln -sf /usr/local/lib/lib$(NAME).so.$(LIB_VER) /usr/local/lib/lib$(NAME).so
+       ldconfig
+       depmod
+       make unload
+       make load
+
+uninstall:
+       rm -f /usr/local/lib/lib$(NAME).so*
+       rm -f /usr/local/include/$(LIB_HDR)
+       rm -f /usr/local/include/$(DRVR_HDR)
+       rm -f /etc/ld.so.conf.d/$(NAME).conf
+       rm -rf /lib/modules/$(KVER)/kernel/drivers/$(NAME)
+       rm -f /etc/udev/rules.d/99-$(NAME).rules
+       if [ -f "$(RHR)" ]; then rm -f /etc/sysconfig/modules/$(NAME).modules;\
+       else cp /etc/modules ./etc.modules.bak; sed -i '/$(NAME)/d' /etc/modules; fi
+       ldconfig
+       depmod
+
+
diff --git a/drivers/staging/riffa/README.txt b/drivers/staging/riffa/README.txt
new file mode 100644
index 000000000000..36e9f620d358
--- /dev/null
+++ b/drivers/staging/riffa/README.txt
@@ -0,0 +1,38 @@
+You must build the Linux driver against the version of the Linux kernel you have
+installed. This will require the Linux kernel headers. After you've built the
+driver you can install it in your system so that it loads at boot time. If the
+driver is installed and there is a RIFFA 2.0.1 capable FPGA installed as well,
+the driver will detect it. Output in the system log will provide additional
+information. This makefile will also build and install the C/C++ native library.
+
+Ensure you have the kernel headers installed:
+
+sudo make setup
+
+This will attempt to install the kernel headers using your system's package
+manager. You can skip this step if you've already installed the kernel headers.
+
+Compile the driver and C/C++ library:
+
+make
+
+or
+
+make debug
+
+Using make debug will compile in code to output debug messages to the system log
+at runtime. These messages are useful when developing your design. However they
+pollute your system log and incur some overhead. So you may want to install the
+non-debug version after you've completed development.
+
+Install the driver and library:
+
+sudo make install
+
+The system will be configured to load the driver at boot time. The C/C++ library
+will be installed in the default library path. The header files will be placed
+in the default include path. You will want to reboot after you've installed for
+the driver to be (re)loaded.
+
+When compiling an application you should only need to include the <riffa.h>
+header file and link with -lriffa.
diff --git a/drivers/staging/riffa/circ_queue.c b/drivers/staging/riffa/circ_queue.c
new file mode 100644
index 000000000000..fb43ca22e3c0
--- /dev/null
+++ b/drivers/staging/riffa/circ_queue.c
@@ -0,0 +1,188 @@
+// ----------------------------------------------------------------------
+// Copyright (c) 2016, The Regents of the University of California All
+// rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//
+//     * Redistributions in binary form must reproduce the above
+//       copyright notice, this list of conditions and the following
+//       disclaimer in the documentation and/or other materials provided
+//       with the distribution.
+//
+//     * Neither the name of The Regents of the University of California
+//       nor the names of its contributors may be used to endorse or
+//       promote products derived from this software without specific
+//       prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL REGENTS OF THE
+// UNIVERSITY OF CALIFORNIA BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+// DAMAGE.
+// ----------------------------------------------------------------------
+
+/*
+ * Filename: circ_queue.c
+ * Version: 1.0
+ * Description: A lock-free single-producer circular queue implementation
+ *   modeled after the more elaborate C++ version from Faustino Frechilla at:
+ *   http://www.codeproject.com/Articles/153898/Yet-another-implementation-of-a-lock-free-circular
+ * Author: Matthew Jacobsen
+ * History: @mattj: Initial release. Version 1.0.
+ */
+
+#include <linux/slab.h>
+#include "circ_queue.h"
+
+circ_queue * init_circ_queue(int len)
+{
+       int i;
+       circ_queue * q;
+
+       q = kzalloc(sizeof(circ_queue), GFP_KERNEL);
+       if (q == NULL) {
+               printk(KERN_ERR "Not enough memory to allocate circ_queue");
+               return NULL;
+       }
+
+       atomic_set(&q->writeIndex, 0);
+       atomic_set(&q->readIndex, 0);
+       q->len = len;
+
+       q->vals = (unsigned int**) kzalloc(len*sizeof(unsigned int*), GFP_KERNEL);
+       if (q->vals == NULL) {
+               printk(KERN_ERR "Not enough memory to allocate circ_queue array");
+               return NULL;
+       }
+       for (i = 0; i < len; i++) {
+               q->vals[i] = (unsigned int*) kzalloc(2*sizeof(unsigned int), GFP_KERNEL);
+               if (q->vals[i] == NULL) {
+                       printk(KERN_ERR "Not enough memory to allocate circ_queue array position");
+                       return NULL;
+               }
+       }
+
+       return q;
+}
+
+/**
+ * Internal function to help count. Returns the queue size normalized position.
+ */
+unsigned int queue_count_to_index(unsigned int count, unsigned int len)
+{
+       return (count % len);
+}
+
+int push_circ_queue(circ_queue * q, unsigned int val1, unsigned int val2)
+{
+       unsigned int currReadIndex;
+       unsigned int currWriteIndex;
+
+       currWriteIndex = atomic_read(&q->writeIndex);
+       currReadIndex  = atomic_read(&q->readIndex);
+       if (queue_count_to_index(currWriteIndex+1, q->len) == queue_count_to_index(currReadIndex, q->len)) {
+               // The queue is full
+               return 1;
+       }
+
+       // Save the data into the queue
+       q->vals[queue_count_to_index(currWriteIndex, q->len)][0] = val1;
+       q->vals[queue_count_to_index(currWriteIndex, q->len)][1] = val2;
+       // Increment atomically write index. Now a consumer thread can read
+       // the piece of data that was just stored.
+       atomic_inc(&q->writeIndex);
+
+       return 0;
+}
+
+int pop_circ_queue(circ_queue * q, unsigned int * val1, unsigned int * val2)
+{
+       unsigned int currReadIndex;
+       unsigned int currMaxReadIndex;
+
+       do
+       {
+               currReadIndex = atomic_read(&q->readIndex);
+               currMaxReadIndex = atomic_read(&q->writeIndex);
+               if (queue_count_to_index(currReadIndex, q->len) == queue_count_to_index(currMaxReadIndex, q->len)) {
+                       // The queue is empty or a producer thread has allocate space in the queue
+                       // but is waiting to commit the data into it
+                       return 1;
+               }
+
+               // Retrieve the data from the queue
+               *val1 = q->vals[queue_count_to_index(currReadIndex, q->len)][0];
+               *val2 = q->vals[queue_count_to_index(currReadIndex, q->len)][1];
+
+               // Try to perfrom now the CAS operation on the read index. If we succeed
+               // label & val already contain what q->readIndex pointed to before we
+               // increased it.
+               if (atomic_cmpxchg(&q->readIndex, currReadIndex, currReadIndex+1) == currReadIndex) {
+                       // The lable & val were retrieved from the queue. Note that the
+                       // data inside the label or value arrays are not deleted.
+                       return 0;
+               }
+
+               // Failed to retrieve the elements off the queue. Someone else must
+               // have read the element stored at countToIndex(currReadIndex)
+               // before we could perform the CAS operation.
+       } while(1); // keep looping to try again!
+
+       return 1;
+}
+
+int circ_queue_empty(circ_queue * q)
+{
+       unsigned int currReadIndex;
+       unsigned int currMaxReadIndex;
+
+       currReadIndex = atomic_read(&q->readIndex);
+       currMaxReadIndex = atomic_read(&q->writeIndex);
+       if (queue_count_to_index(currReadIndex, q->len) == queue_count_to_index(currMaxReadIndex, q->len)) {
+               // The queue is empty or a producer thread has allocate space in the queue
+               // but is waiting to commit the data into it
+               return 1;
+       }
+       return 0;
+}
+
+int circ_queue_full(circ_queue * q)
+{
+       unsigned int currReadIndex;
+       unsigned int currWriteIndex;
+
+       currWriteIndex = atomic_read(&q->writeIndex);
+       currReadIndex  = atomic_read(&q->readIndex);
+       if (queue_count_to_index(currWriteIndex+1, q->len) == queue_count_to_index(currReadIndex, q->len)) {
+               // The queue is full
+               return 1;
+       }
+       return 0;
+}
+
+void free_circ_queue(circ_queue * q)
+{
+       int i;
+
+       if (q == NULL)
+               return;
+
+       for (i = 0; i < q->len; i++) {
+               kfree(q->vals[i]);
+       }
+       kfree(q->vals);
+       kfree(q);
+}
+
diff --git a/drivers/staging/riffa/circ_queue.h b/drivers/staging/riffa/circ_queue.h
new file mode 100644
index 000000000000..e458cdb6aca8
--- /dev/null
+++ b/drivers/staging/riffa/circ_queue.h
@@ -0,0 +1,96 @@
+// ----------------------------------------------------------------------
+// Copyright (c) 2016, The Regents of the University of California All
+// rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//
+//     * Redistributions in binary form must reproduce the above
+//       copyright notice, this list of conditions and the following
+//       disclaimer in the documentation and/or other materials provided
+//       with the distribution.
+//
+//     * Neither the name of The Regents of the University of California
+//       nor the names of its contributors may be used to endorse or
+//       promote products derived from this software without specific
+//       prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL REGENTS OF THE
+// UNIVERSITY OF CALIFORNIA BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+// DAMAGE.
+// ----------------------------------------------------------------------
+
+/*
+ * Filename: circ_queue.h
+ * Version: 1.0
+ * Description: A lock-free single-producer circular queue implementation
+ *   modeled after the more elaborate C++ version from Faustino Frechilla at:
+ *   http://www.codeproject.com/Articles/153898/Yet-another-implementation-of-a-lock-free-circular
+ * Author: Matthew Jacobsen
+ * History: @mattj: Initial release. Version 1.0.
+ */
+#ifndef CIRC_QUEUE_H
+#define CIRC_QUEUE_H
+
+#include <asm/atomic.h>
+
+/* Struct for the circular queue. */
+struct circ_queue {
+       atomic_t writeIndex;
+       atomic_t readIndex;
+       unsigned int ** vals;
+       unsigned int len;
+};
+typedef struct circ_queue circ_queue;
+
+/**
+ * Initializes a circ_queue with depth/length len. Returns non-NULL on success,
+ * NULL if there was a problem creating the queue.
+ */
+circ_queue * init_circ_queue(int len);
+
+/**
+ * Pushes a pair of unsigned int values into the specified queue at the head.
+ * Returns 0 on success, non-zero if there is no more space in the queue.
+ */
+int push_circ_queue(circ_queue * q, unsigned int val1, unsigned int val2);
+
+/**
+ * Pops a pair of unsigned int values out of the specified queue from the tail.
+ * Returns 0 on success, non-zero if the queue is empty.
+ */
+int pop_circ_queue(circ_queue * q, unsigned int * val1, unsigned int * val2);
+
+/**
+ * Returns 1 if the circ_queue is empty, 0 otherwise. Note, this is not a
+ * synchronized function. If another thread is accessing this circ_queue, the
+ * return value may not be valid.
+ */
+int circ_queue_empty(circ_queue * q);
+
+/**
+ * Returns 1 if the circ_queue is full, 0 otherwise. Note, this is not a
+ * synchronized function. If another thread is accessing this circ_queue, the
+ * return value may not be valid.
+ */
+int circ_queue_full(circ_queue * q);
+
+/**
+ * Frees the resources associated with the specified circ_queue.
+ */
+void free_circ_queue(circ_queue * q);
+
+#endif
diff --git a/drivers/staging/riffa/riffa.c b/drivers/staging/riffa/riffa.c
new file mode 100644
index 000000000000..91b9b9af3dc8
--- /dev/null
+++ b/drivers/staging/riffa/riffa.c
@@ -0,0 +1,152 @@
+// ----------------------------------------------------------------------
+// Copyright (c) 2016, The Regents of the University of California All
+// rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//
+//     * Redistributions in binary form must reproduce the above
+//       copyright notice, this list of conditions and the following
+//       disclaimer in the documentation and/or other materials provided
+//       with the distribution.
+//
+//     * Neither the name of The Regents of the University of California
+//       nor the names of its contributors may be used to endorse or
+//       promote products derived from this software without specific
+//       prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL REGENTS OF THE
+// UNIVERSITY OF CALIFORNIA BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+// DAMAGE.
+// ----------------------------------------------------------------------
+
+/*
+ * Filename: riffa.c
+ * Version: 2.0
+ * Description: Linux PCIe communications API for RIFFA.
+ * Author: Matthew Jacobsen
+ * History: @mattj: Initial release. Version 2.0.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <fcntl.h>
+#include "riffa.h"
+
+struct thread_info {    /* Used as argument to thread_start() */
+       // please refer to API of fpga_send() and fpga_recv() at http://riffa.ucsd.edu/node/10 or https://github.com/KastnerRG/riffa/blob/master/driver/linux/riffa.c#L84-L111
+       fpga_t * fpga;
+       unsigned int chnl;
+       unsigned int * buffer;
+       unsigned int len;
+       unsigned int offset;
+       unsigned int last;
+       long long timeout;
+};
+
+struct fpga_t
+{
+       int fd;
+       int id;
+};
+
+fpga_t * fpga_open(int id)
+{
+       fpga_t * fpga;
+
+       // Allocate space for the fpga_dev
+       fpga = (fpga_t *)malloc(sizeof(fpga_t));
+       if (fpga == NULL)
+               return NULL;
+       fpga->id = id;
+
+       // Open the device file.
+       fpga->fd = open("/dev/" DEVICE_NAME, O_RDWR | O_SYNC);
+       if (fpga->fd < 0) {
+               free(fpga);
+               return NULL;
+       }
+
+       return fpga;
+}
+
+void fpga_close(fpga_t * fpga)
+{
+       // Close the device file.
+       close(fpga->fd);
+       free(fpga);
+}
+
+//int fpga_send(fpga_t * fpga, int chnl, void * data, int len, int destoff, int last, long long timeout)
+void* fpga_send(void *arg)
+{
+       struct thread_info *tinfo_send = (struct thread_info *) arg;
+
+       fpga_chnl_io io_send;
+
+       io_send.id = tinfo_send->fpga->id;
+       io_send.chnl = tinfo_send->chnl;
+       io_send.len = tinfo_send->len;
+       io_send.offset = tinfo_send->offset;
+       io_send.last = tinfo_send->last;
+       io_send.timeout = tinfo_send->timeout;
+       io_send.data = (char *)(tinfo_send->buffer);
+
+       int number_of_words_sent = ioctl(tinfo_send->fpga->fd, IOCTL_SEND, &io_send);
+
+       pthread_exit((void *)(intptr_t)number_of_words_sent);
+}
+
+//int fpga_recv(fpga_t * fpga, int chnl, void * data, int len, long long timeout)
+void* fpga_recv(void *arg)
+{
+       struct thread_info *tinfo_recv = (struct thread_info *) arg;
+
+       fpga_chnl_io io_recv;
+
+       io_recv.id = tinfo_recv->fpga->id;
+       io_recv.chnl = tinfo_recv->chnl;
+       io_recv.len = tinfo_recv->len;
+       io_recv.timeout = tinfo_recv->timeout;
+       io_recv.data = (char *)(tinfo_recv->buffer);
+
+       int number_of_words_recv = ioctl(tinfo_recv->fpga->fd, IOCTL_RECV, &io_recv);
+
+       pthread_exit((void *)(intptr_t)number_of_words_recv);
+}
+
+void fpga_reset(fpga_t * fpga)
+{
+       ioctl(fpga->fd, IOCTL_RESET, fpga->id);
+}
+
+int fpga_list(fpga_info_list * list) {
+       int fd;
+       int rc;
+
+       fd = open("/dev/" DEVICE_NAME, O_RDWR | O_SYNC);
+       if (fd < 0)
+               return fd;
+       rc = ioctl(fd, IOCTL_LIST, list);
+       close(fd);
+       return rc;
+}
+
+
+
diff --git a/drivers/staging/riffa/riffa.h b/drivers/staging/riffa/riffa.h
new file mode 100644
index 000000000000..a8db183bd6b8
--- /dev/null
+++ b/drivers/staging/riffa/riffa.h
@@ -0,0 +1,121 @@
+// ----------------------------------------------------------------------
+// Copyright (c) 2016, The Regents of the University of California All
+// rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//
+//     * Redistributions in binary form must reproduce the above
+//       copyright notice, this list of conditions and the following
+//       disclaimer in the documentation and/or other materials provided
+//       with the distribution.
+//
+//     * Neither the name of The Regents of the University of California
+//       nor the names of its contributors may be used to endorse or
+//       promote products derived from this software without specific
+//       prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL REGENTS OF THE
+// UNIVERSITY OF CALIFORNIA BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+// DAMAGE.
+// ----------------------------------------------------------------------
+
+/*
+ * Filename: riffa.h
+ * Version: 2.0
+ * Description: Linux PCIe communications API for RIFFA.
+ * Author: Matthew Jacobsen
+ * History: @mattj: Initial release. Version 2.0.
+ */
+
+#ifndef RIFFA_H
+#define RIFFA_H
+
+#include "riffa_driver.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct fpga_t;
+typedef struct fpga_t fpga_t;
+
+/**
+ * Populates the fpga_info_list pointer with all FPGAs registered in the system.
+ * Returns 0 on success, a negative value on error.
+ */
+int fpga_list(fpga_info_list * list);
+
+/**
+ * Initializes the FPGA specified by id. On success, returns a pointer to a
+ * fpga_t struct. On error, returns NULL. Each FPGA must be opened before any
+ * channels can be accessed. Once opened, any number of threads can use the
+ * fpga_t struct.
+ */
+fpga_t * fpga_open(int id);
+
+/**
+ * Cleans up memory/resources for the FPGA specified by the fd descriptor.
+ */
+void fpga_close(fpga_t * fpga);
+
+/**
+ * Sends len words (4 byte words) from data to FPGA channel chnl using the
+ * fpga_t struct. The FPGA channel will be sent len, destoff, and last. If last
+ * is 1, the channel should interpret the end of this send as the end of a
+ * transaction. If last is 0, the channel should wait for additional sends
+ * before the end of the transaction. If timeout is non-zero, this call will
+ * send data and wait up to timeout ms for the FPGA to respond (between
+ * packets) before timing out. If timeout is zero, this call may block
+ * indefinitely. Multiple threads sending on the same channel may result in
+ * corrupt data or error. This function is thread safe across channels.
+ * On success, returns the number of words sent. On error returns a negative
+ * value.
+ */
+//int fpga_send(fpga_t * fpga, int chnl, void * data, int len, int destoff, int last, long long timeout);
+void* fpga_send(void * arg);
+
+/**
+ * Receives data from the FPGA channel chnl to the data pointer, using the
+ * fpga_t struct. The FPGA channel can send any amount of data, so the data
+ * array should be large enough to accommodate. The len parameter specifies the
+ * actual size of the data buffer in words (4 byte words). The FPGA channel will
+ * specify an offset which will determine where in the data array the data will
+ * start being written. If the amount of data (plus offset) exceed the size of
+ * the data array (len), then that data will be discarded. If timeout is
+ * non-zero, this call will wait up to timeout ms for the FPGA to respond
+ * (between packets) before timing out. If timeout is zero, this call may block
+ * indefinitely. Multiple threads receiving on the same channel may result in
+ * corrupt data or error. This function is thread safe across channels.
+ * On success, returns the number of words written to the data array. On error
+ * returns a negative value.
+ */
+//int fpga_recv(fpga_t * fpga, int chnl, void * data, int len, long long timeout);
+void* fpga_recv(void * arg);
+
+/**
+ * Resets the state of the FPGA and all transfers across all channels. This is
+ * meant to be used as an alternative to rebooting if an error occurs while
+ * sending/receiving. Calling this function while other threads are sending or
+ * receiving will result in unexpected behavior.
+ */
+void fpga_reset(fpga_t * fpga);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/drivers/staging/riffa/riffa_driver.c b/drivers/staging/riffa/riffa_driver.c
new file mode 100644
index 000000000000..e668d316899f
--- /dev/null
+++ b/drivers/staging/riffa/riffa_driver.c
@@ -0,0 +1,1633 @@
+// ----------------------------------------------------------------------
+// Copyright (c) 2016, The Regents of the University of California All
+// rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//
+//     * Redistributions in binary form must reproduce the above
+//       copyright notice, this list of conditions and the following
+//       disclaimer in the documentation and/or other materials provided
+//       with the distribution.
+//
+//     * Neither the name of The Regents of the University of California
+//       nor the names of its contributors may be used to endorse or
+//       promote products derived from this software without specific
+//       prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL REGENTS OF THE
+// UNIVERSITY OF CALIFORNIA BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+// DAMAGE.
+// ----------------------------------------------------------------------
+
+/*
+ * Filename: riffa_driver.c
+ * Version: 2.0
+ * Description: Linux PCIe device driver for RIFFA. Uses Linux kernel APIs in
+ *  version 2.6.27+ (tested on version 2.6.32 - 3.3.0).
+ * Author: Matthew Jacobsen
+ * History: @mattj: Initial release. Version 2.0.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/version.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/fs.h>
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,11,0)
+#include <linux/sched.h>
+#else
+#include <linux/sched/signal.h>
+#endif
+
+#include <linux/rwsem.h>
+#include <linux/dma-mapping.h>
+#include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <asm/uaccess.h>
+#include <asm/div64.h>
+#include "riffa_driver.h"
+#include "circ_queue.h"
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("PCIe driver for RIFFA, Linux (2.6.27+)");
+MODULE_AUTHOR("Matt Jacobsen, Patrick Lai");
+
+#ifndef __devinit
+#define __devinit
+#define __devexit
+#define __devexit_p
+#endif
+
+#define CHNL_REG(c, o) ((c<<4) + o)
+#if !defined(__LP64__) && !defined(_LP64)
+#define BUILD_32 1
+#endif
+
+#define CHNL_FLAG_BUSY 0
+
+struct sg_mapping {
+       struct page ** pages;
+       struct scatterlist * sgl;
+       enum dma_data_direction direction;
+       int num_sg;
+       unsigned long num_pages;
+       unsigned long long length;
+       unsigned long long overflow;
+};
+
+struct chnl_dir {
+       volatile unsigned long flags;
+       wait_queue_head_t waitq;
+       struct circ_queue * msgs;
+       void * buf_addr;
+       dma_addr_t buf_hw_addr;
+       struct sg_mapping * sg_map_0;
+       struct sg_mapping * sg_map_1;
+};
+
+struct fpga_state {
+       struct pci_dev * dev;
+       unsigned long long irq;
+       void __iomem *bar0;
+       unsigned long long bar0_addr;
+       unsigned long long bar0_len;
+       unsigned long long bar0_flags;
+       atomic_t intr_disabled;
+       void * spill_buf_addr;
+       dma_addr_t spill_buf_hw_addr;
+       int num_sg;
+       int sg_buf_size;
+       int id;
+       char name[16];
+       int vendor_id;
+       int device_id;
+       int num_chnls;
+       struct chnl_dir ** recv;
+       struct chnl_dir ** send;
+};
+
+// Global variables (to this file only)
+static struct class * mymodule_class;
+static dev_t devt;
+static atomic_t used_fpgas[NUM_FPGAS];
+static struct fpga_state * fpgas[NUM_FPGAS];
+
+static unsigned int tx_len;
+static bool recv_sg_buf_populated;
+
+///////////////////////////////////////////////////////
+// MEMORY ALLOCATION & HELPER FUNCTIONS
+///////////////////////////////////////////////////////
+
+/**
+ * Returns the value at the specified address.
+ */
+static inline unsigned int read_reg(struct fpga_state * sc, int offset)
+{
+       return readl(sc->bar0 + (offset<<2));
+}
+
+/**
+ * Writes the value to the specified address.
+ */
+static inline void write_reg(struct fpga_state * sc, int offset, unsigned int val)
+{
+       writel(val, sc->bar0 + (offset<<2));
+}
+
+#ifdef BUILD_32
+/**
+ * Needed for 32 bit OS because dma_map_sg macro eventually does some 64 bit
+ * division.
+ */
+unsigned long long __udivdi3(unsigned long long num, unsigned long long den)
+{
+       do_div(num, den);
+       return num;
+}
+#endif
+
+
+// These are not defined in the 2.x.y kernels, so just define them
+#if LINUX_VERSION_CODE <= KERNEL_VERSION(2,6,39)
+#define PCI_EXP_DEVCTL2_IDO_REQ_EN 0x100
+#define PCI_EXP_DEVCTL2_IDO_CMP_EN 0x200
+#else
+/**
+ * These are badly named in pre-3.6.11 kernel versions.  We COULD do the same
+ * check as above, however (annoyingly) linux for tegra (based on post-3.6.11)
+ * picked up the header file from some pre-3.6.11 version, so we'll just make
+ * our code ugly and handle the check here:
+ */
+#ifndef PCI_EXP_DEVCTL2_IDO_REQ_EN
+#define PCI_EXP_DEVCTL2_IDO_REQ_EN PCI_EXP_IDO_REQ_EN
+#endif
+#ifndef PCI_EXP_DEVCTL2_IDO_CMP_EN
+#define PCI_EXP_DEVCTL2_IDO_CMP_EN PCI_EXP_IDO_CMP_EN
+#endif
+#endif
+
+#if LINUX_VERSION_CODE <= KERNEL_VERSION(3,6,11)
+/**
+ * Code used to set ETB and RCB, but not available before 3.0, or incorrectly
+ * defined before 3.7. As it is peppered throughout the clean up code, it's just
+ * easier to copy the declarations (not verbatim) here than a bunch of conditionals
+ * everywhere else.
+ */
+
+int pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *val)
+{
+       int ret;
+
+       *val = 0;
+       if (pos & 1)
+               return -EINVAL;
+
+       ret = pci_read_config_word(dev, pci_pcie_cap(dev) + pos, val);
+       /*
+        * Reset *val to 0 if pci_read_config_word() fails, it may
+        * have been written as 0xFFFF if hardware error happens
+        * during pci_read_config_word().
+        */
+       if (ret)
+               *val = 0;
+       return ret;
+}
+
+int pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *val)
+{
+       int ret;
+
+       *val = 0;
+       if (pos & 3)
+               return -EINVAL;
+
+       ret = pci_read_config_dword(dev, pci_pcie_cap(dev) + pos, val);
+       /*
+        * Reset *val to 0 if pci_read_config_dword() fails, it may
+        * have been written as 0xFFFFFFFF if hardware error happens
+        * during pci_read_config_dword().
+        */
+       if (ret)
+               *val = 0;
+       return ret;
+
+}
+
+int pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val)
+{
+       if (pos & 1)
+               return -EINVAL;
+
+       return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val);
+}
+
+int pcie_capability_write_dword(struct pci_dev *dev, int pos, u32 val)
+{
+       if (pos & 3)
+               return -EINVAL;
+
+       return pci_write_config_dword(dev, pci_pcie_cap(dev) + pos, val);
+}
+#endif
+
+
+
+
+///////////////////////////////////////////////////////
+// INTERRUPT HANDLER
+///////////////////////////////////////////////////////
+
+/**
+ * Reads the interrupt vector and processes it. If processing VECT0, off will
+ * be 0. If processing VECT1, off will be 6.
+ */
+static inline void process_intr_vector(struct fpga_state * sc, int off,
+                               unsigned int vect)
+{
+       // VECT_0/VECT_1 are organized from right to left (LSB to MSB) as:
+       // [ 0] TX_TXN                  for channel 0 in VECT_0, channel 6 in VECT_1
+       // [ 1] TX_SG_BUF_RECVD for channel 0 in VECT_0, channel 6 in VECT_1
+       // [ 2] TX_TXN_DONE             for channel 0 in VECT_0, channel 6 in VECT_1
+       // [ 3] RX_SG_BUF_RECVD for channel 0 in VECT_0, channel 6 in VECT_1
+       // [ 4] RX_TXN_DONE             for channel 0 in VECT_0, channel 6 in VECT_1
+       // ...
+       // [25] TX_TXN                  for channel 5 in VECT_0, channel 11 in VECT_1
+       // [26] TX_SG_BUF_RECVD for channel 5 in VECT_0, channel 11 in VECT_1
+       // [27] TX_TXN_DONE             for channel 5 in VECT_0, channel 11 in VECT_1
+       // [28] RX_SG_BUF_RECVD for channel 5 in VECT_0, channel 11 in VECT_1
+       // [29] RX_TXN_DONE             for channel 5 in VECT_0, channel 11 in VECT_1
+       // Positions 30 - 31 in both VECT_0 and VECT_1 are zero.
+
+       unsigned int offlast;
+       unsigned int len;
+       int recv;
+       int send;
+       int chnl;
+       int i;
+
+//printk(KERN_INFO "riffa: intrpt_handler received:%08x\n", vect);
+       if (vect & 0xC0000000) {
+               printk(KERN_ERR "riffa: fpga:%d, received bad interrupt vector:%08x\n", sc->id, vect);
+               return;
+       }
+
+       for (i = 0; i < 6 && (i+off) < sc->num_chnls; ++i) {
+               chnl = i + off;
+               recv = 0;
+               send = 0;
+
+               // TX (PC receive) scatter gather buffer is read.
+               if (vect & (1<<((5*i)+1))) {
+                       recv = 1;
+                       // Keep track so the thread can handle this.
+                       if (push_circ_queue(sc->recv[chnl]->msgs, EVENT_SG_BUF_READ, 0)) {
+                               printk(KERN_ERR "riffa: fpga:%d chnl:%d, recv sg buf read msg queue full\n", sc->id, chnl);
+                       }
+                       DEBUG_MSG(KERN_INFO "riffa: fpga:%d chnl:%d, recv sg buf read\n", sc->id, chnl);
+               }
+
+               // TX (PC receive) transaction done.
+               if (vect & (1<<((5*i)+2))) {
+                       recv = 1;
+                       // Read the transferred amount.
+                       len = read_reg(sc, CHNL_REG(chnl, TX_TNFR_LEN_REG_OFF));
+                       // Notify the thread.
+                       if (push_circ_queue(sc->recv[chnl]->msgs, EVENT_TXN_DONE, len)) {
+                               printk(KERN_ERR "riffa: fpga:%d chnl:%d, recv txn done msg queue full\n", sc->id, chnl);
+                       }
+                       DEBUG_MSG(KERN_INFO "riffa: fpga:%d chnl:%d, recv txn done\n", sc->id, chnl);
+               }
+
+               // New TX (PC receive) transaction.
+               if (vect & (1<<((5*i)+0))) {
+                       recv = 1;
+                       recv_sg_buf_populated = 0; // resets for new transaction
+
+                       // Read the offset/last and length
+                       offlast = read_reg(sc, CHNL_REG(chnl, TX_OFFLAST_REG_OFF));
+                       tx_len = read_reg(sc, CHNL_REG(chnl, TX_LEN_REG_OFF));
+                       // Keep track of this transaction
+                       if (push_circ_queue(sc->recv[chnl]->msgs, EVENT_TXN_OFFLAST, offlast)) {
+                               printk(KERN_ERR "riffa: fpga:%d chnl:%d, recv txn offlast msg queue full\n", sc->id, chnl);
+                       }
+                       /*if (push_circ_queue(sc->recv[chnl]->msgs, EVENT_TXN_LEN, len)) {
+                               printk(KERN_ERR "riffa: fpga:%d chnl:%d, recv txn len msg queue full\n", sc->id, chnl);
+                       }*/
+                       DEBUG_MSG(KERN_INFO "riffa: fpga:%d chnl:%d, recv txn (len:%d off:%d last:%d)\n", sc->id, chnl, tx_len, (offlast>>1), (offlast & 0x1));
+               }
+
+               // RX (PC send) scatter gather buffer is read.
+               if (vect & (1<<((5*i)+3))) {
+                       send = 1;
+                       // Keep track so the thread can handle this.
+                       if (push_circ_queue(sc->send[chnl]->msgs, EVENT_SG_BUF_READ, 0)) {
+                               printk(KERN_ERR "riffa: fpga:%d chnl:%d, send sg buf read msg queue full\n", sc->id, chnl);
+                       }
+                       DEBUG_MSG(KERN_INFO "riffa: fpga:%d chnl:%d, send sg buf read\n", sc->id, chnl);
+               }
+
+               // RX (PC send) transaction done.
+               if (vect & (1<<((5*i)+4))) {
+                       send = 1;
+                       // Read the transferred amount.
+                       len = read_reg(sc, CHNL_REG(chnl, RX_TNFR_LEN_REG_OFF));
+                       // Notify the thread.
+                       if (push_circ_queue(sc->send[chnl]->msgs, EVENT_TXN_DONE, len)) {
+                               printk(KERN_ERR "riffa: fpga:%d chnl:%d, send txn done msg queue full\n", sc->id, chnl);
+                       }
+                       DEBUG_MSG(KERN_INFO "riffa: fpga:%d chnl:%d, send txn done\n", sc->id, chnl);
+               }
+
+               // Wake up the thread?
+               if (recv)
+                       wake_up(&sc->recv[chnl]->waitq);
+               if (send)
+                       wake_up(&sc->send[chnl]->waitq);
+       }
+}
+
+/**
+ * Interrupt handler for all interrupts on all files. Reads data/values
+ * from FPGA and wakes up waiting threads to process the data. Always returns
+ * IRQ_HANDLED.
+ */
+static irqreturn_t intrpt_handler(int irq, void *dev_id)
+{
+       unsigned int vect0;
+       unsigned int vect1;
+       struct fpga_state * sc;
+
+       sc = (struct fpga_state *)dev_id;
+       vect0 = 0;
+       vect1 = 0;
+
+       if (sc == NULL) {
+               printk(KERN_ERR "riffa: invalid fpga_state pointer\n");
+               return IRQ_HANDLED;
+       }
+
+       if (!atomic_read(&sc->intr_disabled)) {
+               // Read the interrupt vector(s):
+               vect0 = read_reg(sc, IRQ_REG0_OFF);
+               if (sc->num_chnls > 6)
+                       vect1 = read_reg(sc, IRQ_REG1_OFF);
+
+               // Process the vector(s)
+               process_intr_vector(sc, 0, vect0);
+               if (sc->num_chnls > 6)
+                       process_intr_vector(sc, 6, vect1);
+       }
+
+       return IRQ_HANDLED;
+}
+
+
+///////////////////////////////////////////////////////
+// FILE OPERATION HANDLERS
+///////////////////////////////////////////////////////
+
+/**
+ * Creates and returns a struct_sg_mapping that holds all the data for the user
+ * pages that have been mapped into a scatterlist array. Assumes the user data
+ * is 32 bit word aligned. Up to length bytes from the udata pointer will be
+ * mapped. After all length bytes are mapped, up to overflow bytes will be
+ * mapped using the common spill buffer for the channel. The overflow is used
+ * if we run out of space in the supplied udata pointer.
+ */
+static inline struct sg_mapping * fill_sg_buf(struct fpga_state * sc, int chnl,
+                                       void * sg_buf, unsigned long udata, unsigned long long length,
+                                       unsigned long long overflow, enum dma_data_direction direction) {
+       const char * dir = (direction == DMA_TO_DEVICE ? "send" : "recv");
+       struct sg_mapping * sg_map;
+       struct page ** pages = NULL;
+       struct scatterlist * sgl = NULL;
+       struct scatterlist * sg;
+       unsigned long num_pages_reqd = 0;
+       long num_pages = 0;
+       unsigned int fp_offset;
+       unsigned int len;
+       unsigned int hw_len;
+       dma_addr_t hw_addr;
+       unsigned long long len_rem = length;
+       unsigned long long overflow_rem = overflow;
+       unsigned int * sg_buf_ptr = (unsigned int *)sg_buf;
+       int num_sg = 0;
+       int i;
+
+       // Create the sg_mapping struct.
+       if ((sg_map = (struct sg_mapping *)kmalloc(sizeof(*sg_map), GFP_KERNEL)) == NULL) {
+               printk(KERN_ERR "riffa: fpga:%d chnl:%d, %s could not allocate memory for sg_mapping struct\n", sc->id, chnl, dir);
+               return NULL;
+       }
+
+       if (length > 0) {
+               // Create the pages array.
+               num_pages_reqd = ((udata + length - 1)>>PAGE_SHIFT) - (udata>>PAGE_SHIFT) + 1;
+               num_pages_reqd = (num_pages_reqd > sc->num_sg ? sc->num_sg : num_pages_reqd);
+               if ((pages = kmalloc(num_pages_reqd * sizeof(*pages), GFP_KERNEL)) == NULL) {
+                       printk(KERN_ERR "riffa: fpga:%d chnl:%d, %s could not allocate memory for pages array\n", sc->id, chnl, dir);
+                       kfree(sg_map);
+                       return NULL;
+               }
+
+               // Page in the user pages.
+               down_read(&current->mm->mmap_sem);
+               #if LINUX_VERSION_CODE < KERNEL_VERSION(4,6,0)
+               num_pages = get_user_pages(current, current->mm, udata, num_pages_reqd, 1, 0, pages, NULL);
+               #elsif LINUX_VERSION_CODE < KERNEL_VERSION(4,9,0)
+               num_pages = get_user_pages(udata, num_pages_reqd, 1, 0, pages, NULL);
+               #else
+               num_pages = get_user_pages(udata, num_pages_reqd, FOLL_WRITE, pages, NULL);
+               #endif
+               up_read(&current->mm->mmap_sem);
+               if (num_pages <= 0) {
+                       printk(KERN_ERR "riffa: fpga:%d chnl:%d, %s unable to pin any pages in memory\n", sc->id, chnl, dir);
+                       kfree(pages);
+                       kfree(sg_map);
+                       return NULL;
+               }
+
+               // Create the scatterlist array.
+               if ((sgl = kcalloc(num_pages, sizeof(*sgl), GFP_KERNEL)) == NULL) {
+                       printk(KERN_ERR "riffa: fpga:%d chnl:%d, %s could not allocate memory for scatterlist array\n", sc->id, chnl, dir);
+                       for (i = 0; i < num_pages; ++i)
+                               #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
+                               page_cache_release(pages[i]);
+                               #else
+                               put_page(pages[i]);
+                               #endif
+                       kfree(pages);
+                       kfree(sg_map);
+                       return NULL;
+               }
+
+               // Set the scatterlist values
+               fp_offset = (udata & (~PAGE_MASK));
+               sg_init_table(sgl, num_pages);
+               for (i = 0; i < num_pages; ++i) {
+                       len = ((fp_offset + len_rem) > PAGE_SIZE ? (PAGE_SIZE - fp_offset) : len_rem);
+                       sg_set_page(&sgl[i], pages[i], len, fp_offset);
+                       len_rem -= len;
+                       fp_offset = 0;
+               }
+
+               // Map the scatterlist values and write to the common buffer area
+               num_sg = dma_map_sg(&sc->dev->dev, sgl, num_pages, direction);
+               for_each_sg(sgl, sg, num_sg, i) {
+                       hw_addr = sg_dma_address(sg);
+                       hw_len = sg_dma_len(sg);
+                       sg_buf_ptr[(i*4)+0] = (hw_addr & 0xFFFFFFFF);
+                       sg_buf_ptr[(i*4)+1] = ((hw_addr>>32) & 0xFFFFFFFF);
+                       sg_buf_ptr[(i*4)+2] = hw_len>>2; // Words!
+               }
+       }
+
+       // Provide scatter gather mappings for overflow data (all in spill common buffer)
+       while (len_rem == 0 && overflow_rem > 0 && num_sg < sc->num_sg) {
+               sg_buf_ptr[(num_sg*4)+0] = (sc->spill_buf_hw_addr & 0xFFFFFFFF);
+               sg_buf_ptr[(num_sg*4)+1] = ((sc->spill_buf_hw_addr>>32) & 0xFFFFFFFF);
+               sg_buf_ptr[(num_sg*4)+2] = SPILL_BUF_SIZE>>2; // Words!
+               num_sg++;
+               overflow_rem -= (SPILL_BUF_SIZE > overflow ? overflow : SPILL_BUF_SIZE);
+       }
+
+       // Populate the number of bytes mapped and other sg data
+       sg_map->direction = direction;
+       sg_map->num_pages = num_pages;
+       sg_map->num_sg = num_sg;
+       sg_map->length = (length - len_rem);
+       sg_map->overflow = (overflow - overflow_rem);
+       sg_map->pages = pages;
+       sg_map->sgl = sgl;
+
+       return sg_map;
+}
+
+/**
+ * Frees the scatterlist mappings in the struct sg_mapping pointer and frees all
+ * corresponding structs.
+ */
+static inline void free_sg_buf(struct fpga_state * sc, struct sg_mapping * sg_map) {
+       int i;
+
+       if (sg_map == NULL)
+               return;
+
+       // Unmap the pages.
+       if (sg_map->sgl != NULL)
+               dma_unmap_sg(&sc->dev->dev, sg_map->sgl, sg_map->num_pages, sg_map->direction);
+
+       // Free the pages (mark dirty if necessary).
+       if (sg_map->pages != NULL) {
+               if (sg_map->direction == DMA_FROM_DEVICE) {
+                       for (i = 0; i < sg_map->num_pages; ++i) {
+                               if (!PageReserved(sg_map->pages[i]))
+                                       SetPageDirty(sg_map->pages[i]);
+                               #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
+                               page_cache_release(sg_map->pages[i]);
+                               #else
+                               put_page(sg_map->pages[i]);
+                               #endif
+                       }
+               }
+               else {
+                       for (i = 0; i < sg_map->num_pages; ++i) {
+                               #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)
+                               page_cache_release(sg_map->pages[i]);
+                               #else
+                               put_page(sg_map->pages[i]);
+                               #endif
+                       }
+               }
+       }
+
+       // Free the structures.
+       if (sg_map->pages != NULL)
+               kfree(sg_map->pages);
+       if (sg_map->sgl != NULL)
+               kfree(sg_map->sgl);
+       kfree(sg_map);
+}
+
+/**
+ * Reads data from the FPGA. Will block until all the data is received from the
+ * FPGA unless timeout is non-zero. If timeout is non-zero, the function will
+ * block until all the data is received or until the timeout expires. Received
+ * data will be written directly into the user buffer, bufp, by the DMA process
+ * (using scatter gather). Up to len words (each word == 32 bits) will be
+ * written. On success, the number of words received are returned. On error,
+ * returns a negative value.
+ */
+static inline unsigned int chnl_recv(struct fpga_state * sc, int chnl,
+                               char  __user * bufp, unsigned int len, unsigned long long timeout)
+{
+       struct sg_mapping * sg_map;
+       long tymeouto;
+       long tymeout;
+       int nomsg;
+       unsigned int msg_type;
+       unsigned int msg;
+       int last = -1;
+       unsigned long long offset = 0;
+       unsigned long long length = 0;
+       unsigned long long overflow = 0;
+       unsigned long long capacity = (((unsigned long long)len)<<2);
+       unsigned long long recvd = 0;
+       unsigned long udata = (unsigned long)bufp;
+       unsigned long max_ptr;
+
+       DEFINE_WAIT(wait);
+
+       // Convert timeout to jiffies.
+       tymeout = (timeout == 0 ? MAX_SCHEDULE_TIMEOUT : (timeout * HZ/1000 > LONG_MAX ? LONG_MAX : timeout * HZ/1000));
+       tymeouto = tymeout;
+
+       // Initialize the sg_maps
+       sc->recv[chnl]->sg_map_0 = NULL;
+       sc->recv[chnl]->sg_map_1 = NULL;
+
+       // Continue until we get a message or timeout.
+       while (1) {
+               while ((nomsg = pop_circ_queue(sc->recv[chnl]->msgs, &msg_type, &msg))) {
+                       prepare_to_wait(&sc->recv[chnl]->waitq, &wait, TASK_INTERRUPTIBLE);
+                       // Another check before we schedule.
+                       if ((nomsg = pop_circ_queue(sc->recv[chnl]->msgs, &msg_type, &msg)))
+                               tymeout = schedule_timeout(tymeout);
+                       finish_wait(&sc->recv[chnl]->waitq, &wait);
+                       if (signal_pending(current)) {
+                               free_sg_buf(sc, sc->recv[chnl]->sg_map_0);
+                               free_sg_buf(sc, sc->recv[chnl]->sg_map_1);
+                               return -ERESTARTSYS;
+                       }
+                       if (!nomsg)
+                               break;
+                       if (tymeout == 0) {
+                               printk(KERN_ERR "riffa: fpga:%d chnl:%d, recv timed out\n", sc->id, chnl);
+                               /*free_sg_buf(sc, sc->recv[chnl]->sg_map_0);
+                               free_sg_buf(sc, sc->recv[chnl]->sg_map_1);
+                               return (unsigned int)(recvd>>2);*/
+                       }
+               }
+               tymeout = tymeouto;
+               DEBUG_MSG(KERN_INFO "msg_type: %d\n", msg_type); // added by cheng fei
+               // Process the message.
+               switch (msg_type) {
+               case EVENT_TXN_OFFLAST:
+                       // Read the offset and last flags (always before reading length)
+                       offset = (((unsigned long long)(msg>>1))<<2);
+                       last = (msg & 0x1);
+                       //break;
+
+               //case EVENT_TXN_LEN:
+                       // Read the length
+                       //length = (((unsigned long long)msg)<<2);
+                       length = tx_len << 2;
+                       recvd = 0;
+                       overflow = 0;
+                       // Check for address overflow
+                       max_ptr = (unsigned long)(udata + offset + length - 1);
+                       if (max_ptr < udata) {
+                               printk(KERN_ERR "riffa: fpga:%d chnl:%d, recv pointer address overflow\n", sc->id, chnl);
+                               overflow = length;
+                               length = 0;
+                       }
+                       // Check for capacity overflow
+                       if ((offset + length) > capacity) {
+                               if (offset > capacity) {
+                                       overflow = length;
+                                       length = 0;
+                               }
+                               else {
+                                       overflow = length + offset - capacity;
+                                       length = capacity - offset;
+                               }
+                       }
+                       // Use the recv common buffer to share the scatter gather elements.
+                       if (length > 0 || overflow > 0) {
+                               udata = udata + offset;
+                               sg_map = fill_sg_buf(sc, chnl, sc->recv[chnl]->buf_addr, udata, length, overflow, DMA_FROM_DEVICE);
+                               if (sg_map == NULL || sg_map->num_sg == 0)
+                                       return (unsigned int)(recvd>>2);
+                               // Update based on the sg_mapping
+                               udata += sg_map->length;
+                               length -= sg_map->length;
+                               overflow -= sg_map->overflow;
+                               sc->recv[chnl]->sg_map_1 = sg_map;
+                               // Let FPGA know about the scatter gather buffer.
+                               write_reg(sc, CHNL_REG(chnl, TX_SG_ADDR_LO_REG_OFF), (sc->recv[chnl]->buf_hw_addr & 0xFFFFFFFF));
+                               write_reg(sc, CHNL_REG(chnl, TX_SG_ADDR_HI_REG_OFF), ((sc->recv[chnl]->buf_hw_addr>>32) & 0xFFFFFFFF));
+                               write_reg(sc, CHNL_REG(chnl, TX_SG_LEN_REG_OFF), 4 * sg_map->num_sg);
+
+                               recv_sg_buf_populated = 1;
+
+                               DEBUG_MSG(KERN_INFO "riffa: fpga:%d chnl:%d, recv sg buf populated, %d sent\n", sc->id, chnl, sg_map->num_sg);
+
+                               wake_up(&sc->send[chnl]->waitq);  // https://elixir.bootlin.com/linux/v4.19-rc7/source/include/linux/wait.h#L476
+                               // The @condition is checked each time the waitqueue @wq_head is woken up. wake_up() has to be called after changing any variable that could change the result of the wait condition.
+                       }
+                       break;
+
+               case EVENT_SG_BUF_READ:
+                       // Ignore if we haven't received offlast/len.
+                       if (last == -1)
+                               break;
+                       // Release the previous scatter gather data.
+                       if (sc->recv[chnl]->sg_map_0 != NULL)
+                               recvd += sc->recv[chnl]->sg_map_0->length;
+                       free_sg_buf(sc, sc->recv[chnl]->sg_map_0);
+                       sc->recv[chnl]->sg_map_0 = NULL;
+                       // Populate the common buffer with more scatter gather data?
+                       if (length > 0 || overflow > 0) {
+                               sg_map = fill_sg_buf(sc, chnl, sc->recv[chnl]->buf_addr, udata, length, overflow, DMA_FROM_DEVICE);
+                               if (sg_map == NULL || sg_map->num_sg == 0) {
+                                       free_sg_buf(sc, sc->recv[chnl]->sg_map_0);
+                                       free_sg_buf(sc, sc->recv[chnl]->sg_map_1);
+                                       return (unsigned int)(recvd>>2);
+                               }
+                               // Update based on the sg_mapping
+                               udata += sg_map->length;
+                               length -= sg_map->length;
+                               overflow -= sg_map->overflow;
+                               sc->recv[chnl]->sg_map_0 = sc->recv[chnl]->sg_map_1;
+                               sc->recv[chnl]->sg_map_1 = sg_map;
+                               write_reg(sc, CHNL_REG(chnl, TX_SG_ADDR_LO_REG_OFF), (sc->recv[chnl]->buf_hw_addr & 0xFFFFFFFF));
+                               write_reg(sc, CHNL_REG(chnl, TX_SG_ADDR_HI_REG_OFF), ((sc->recv[chnl]->buf_hw_addr>>32) & 0xFFFFFFFF));
+                               write_reg(sc, CHNL_REG(chnl, TX_SG_LEN_REG_OFF), 4 * sg_map->num_sg);
+                               DEBUG_MSG(KERN_INFO "riffa: fpga:%d chnl:%d, recv sg buf populated, %d sent\n", sc->id, chnl, sg_map->num_sg);
+                       }
+                       break;
+
+               case EVENT_TXN_DONE:
+                       recv_sg_buf_populated = 0; // resets recv sg buf parameters for next transaction.
+
+                       // Ignore if we haven't received offlast/len.
+                       if (last == -1)
+                               break;
+                       // Update with the true value of words transferred.
+                       recvd = (((unsigned long long)msg)<<2);
+                       // Return if this was the last transaction.
+                       free_sg_buf(sc, sc->recv[chnl]->sg_map_0);
+                       free_sg_buf(sc, sc->recv[chnl]->sg_map_1);
+                       sc->recv[chnl]->sg_map_0 = NULL;
+                       sc->recv[chnl]->sg_map_1 = NULL;
+                       DEBUG_MSG(KERN_INFO "riffa: fpga:%d chnl:%d, received %d words\n", sc->id, chnl, (unsigned int)(recvd>>2));
+                       if (last)
+                               return (unsigned int)(recvd>>2);
+                       break;
+
+               default:
+                       printk(KERN_ERR "riffa: fpga:%d chnl:%d, received unknown msg: %08x\n", sc->id, chnl, msg);
+                       break;
+               }
+       }
+
+       return 0;
+}
+
+static inline unsigned int chnl_recv_wrapcheck(struct fpga_state * sc, int chnl,
+                               char  __user * bufp, unsigned int len, unsigned long long timeout)
+{
+       unsigned long udata = (unsigned long)bufp;
+       unsigned int ret;
+
+       // Validate the parameters.
+       if (chnl >= sc->num_chnls || chnl < 0) {
+               printk(KERN_INFO "riffa: fpga:%d chnl:%d, recv channel invalid!\n", sc->id, chnl);
+               return 0;
+       }
+       if (udata & 0x3) {
+               printk(KERN_INFO "riffa: fpga:%d chnl:%d, recv user buffer must be 32 bit word aligned!\n", sc->id, chnl);
+               return -EINVAL;
+       }
+
+       // Ensure no simultaneous operations from several threads
+       if (test_and_set_bit(CHNL_FLAG_BUSY, &sc->recv[chnl]->flags) != 0) {
+               printk(KERN_ERR "riffa: fpga:%d chnl:%d, recv conflict between threads!\n", sc->id, chnl);
+               return -EBUSY;
+       }
+
+       ret = chnl_recv(sc, chnl, bufp, len, timeout);
+
+       // Clear the busy flag
+       clear_bit(CHNL_FLAG_BUSY, &sc->recv[chnl]->flags);
+
+       return ret;
+}
+
+/**
+ * Writes data to the FPGA channel specified. Will block until all the data is
+ * sent to the FPGA unless a non-zero timeout is configured. If timeout is non-
+ * zero, then the function will block until all data is sent or when the timeout
+ * ms elapses. User data from the bufp pointer will be sent, up to len words
+ * (each word == 32 bits). The channel will be told how much data to expect and
+ * at what offset. If last == 1, the FPGA channel will recognize this
+ * transaction as complete after sending. If last == 0, the FPGA channel will
+ * expect additional transactions. On success, returns the number of words sent.
+ * On error, returns a negative value.
+ */
+static inline unsigned int chnl_send(struct fpga_state * sc, int chnl,
+                               const char  __user * bufp, unsigned int len, unsigned int offset,
+                               unsigned int last, unsigned long long timeout)
+{
+       struct sg_mapping * sg_map;
+       long tymeouto;
+       long tymeout;
+       int nomsg;
+       unsigned int msg_type;
+       unsigned int msg;
+       unsigned long long sent = 0;
+       unsigned long long length = (((unsigned long long)len)<<2);
+       unsigned long udata = (unsigned long)bufp;
+
+       DEFINE_WAIT(wait);
+
+       // Convert timeout to jiffies.
+       tymeout = (timeout == 0 ? MAX_SCHEDULE_TIMEOUT : (timeout * HZ/1000 > LONG_MAX ? LONG_MAX : timeout * HZ/1000));
+       tymeouto = tymeout;
+
+       // Clear the message queue.
+       while (!pop_circ_queue(sc->send[chnl]->msgs, &msg_type, &msg));
+
+       // Initialize the sg_maps
+       sc->send[chnl]->sg_map_0 = NULL;
+       sc->send[chnl]->sg_map_1 = NULL;
+
+       // Let FPGA know about transfer.
+       DEBUG_MSG(KERN_INFO "riffa: fpga:%d chnl:%d, send (len:%d off:%d last:%d)\n", sc->id, chnl, len, offset, last);
+       write_reg(sc, CHNL_REG(chnl, RX_OFFLAST_REG_OFF), ((offset<<1) | last));
+       write_reg(sc, CHNL_REG(chnl, RX_LEN_REG_OFF), len);
+       if (len == 0)
+               return 0;
+
+       // Use the send common buffer to share the scatter gather data
+       sg_map = fill_sg_buf(sc, chnl, sc->send[chnl]->buf_addr, udata, length, 0, DMA_TO_DEVICE);
+       if (sg_map == NULL || sg_map->num_sg == 0)
+               return (unsigned int)(sent>>2);
+
+       // Update based on the sg_mapping
+       udata += sg_map->length;
+       length -= sg_map->length;
+       sc->send[chnl]->sg_map_1 = sg_map;
+
+       if(tx_len > 0) { // FPGA initiates new Tx transaction, so "yield" to software chnl_recv() thread
+
+               // gives time for software chnl_recv() thread to populate recv sg buf parameter
+               wait_event_interruptible_timeout(sc->send[chnl]->waitq, (recv_sg_buf_populated == 1), timeout);
+       }
+
+       // Let FPGA know about the scatter gather buffer.
+       write_reg(sc, CHNL_REG(chnl, RX_SG_ADDR_LO_REG_OFF), (sc->send[chnl]->buf_hw_addr & 0xFFFFFFFF));
+       write_reg(sc, CHNL_REG(chnl, RX_SG_ADDR_HI_REG_OFF), ((sc->send[chnl]->buf_hw_addr>>32) & 0xFFFFFFFF));
+       write_reg(sc, CHNL_REG(chnl, RX_SG_LEN_REG_OFF), 4 * sg_map->num_sg);
+       DEBUG_MSG(KERN_INFO "riffa: fpga:%d chnl:%d, send sg buf populated, %d sent\n", sc->id, chnl, sg_map->num_sg);
+
+       // Continue until we get a message or timeout.
+       while (1) {
+               while ((nomsg = pop_circ_queue(sc->send[chnl]->msgs, &msg_type, &msg))) {
+                       prepare_to_wait(&sc->send[chnl]->waitq, &wait, TASK_INTERRUPTIBLE);
+                       // Another check before we schedule.
+                       if ((nomsg = pop_circ_queue(sc->send[chnl]->msgs, &msg_type, &msg)))
+                               tymeout = schedule_timeout(tymeout);
+                       finish_wait(&sc->send[chnl]->waitq, &wait);
+                       if (signal_pending(current)) {
+                               free_sg_buf(sc, sc->send[chnl]->sg_map_0);
+                               free_sg_buf(sc, sc->send[chnl]->sg_map_1);
+                               return -ERESTARTSYS;
+                       }
+                       if (!nomsg)
+                               break;
+                       if (tymeout == 0) {
+                               printk(KERN_ERR "riffa: fpga:%d chnl:%d, send timed out\n", sc->id, chnl);
+                               /*free_sg_buf(sc, sc->send[chnl]->sg_map_0);
+                               free_sg_buf(sc, sc->send[chnl]->sg_map_1);
+                               return (unsigned int)(sent>>2);*/
+                       }
+               }
+               tymeout = tymeouto;
+
+               // Process the message.
+               switch (msg_type) {
+               case EVENT_SG_BUF_READ:
+                       // Release the previous scatter gather data?
+                       if (sc->send[chnl]->sg_map_0 != NULL)
+                               sent += sc->send[chnl]->sg_map_0->length;
+                       free_sg_buf(sc, sc->send[chnl]->sg_map_0);
+                       sc->send[chnl]->sg_map_0 = NULL;
+                       // Populate the common buffer with more scatter gather data?
+                       if (length > 0) {
+                               sg_map = fill_sg_buf(sc, chnl, sc->send[chnl]->buf_addr, udata, length, 0, DMA_TO_DEVICE);
+                               if (sg_map == NULL || sg_map->num_sg == 0) {
+                                       free_sg_buf(sc, sc->send[chnl]->sg_map_0);
+                                       free_sg_buf(sc, sc->send[chnl]->sg_map_1);
+                                       return (unsigned int)(sent>>2);
+                               }
+                               // Update based on the sg_mapping
+                               udata += sg_map->length;
+                               length -= sg_map->length;
+                               sc->send[chnl]->sg_map_0 = sc->send[chnl]->sg_map_1;
+                               sc->send[chnl]->sg_map_1 = sg_map;
+                               write_reg(sc, CHNL_REG(chnl, RX_SG_ADDR_LO_REG_OFF), (sc->send[chnl]->buf_hw_addr & 0xFFFFFFFF));
+                               write_reg(sc, CHNL_REG(chnl, RX_SG_ADDR_HI_REG_OFF), ((sc->send[chnl]->buf_hw_addr>>32) & 0xFFFFFFFF));
+                               write_reg(sc, CHNL_REG(chnl, RX_SG_LEN_REG_OFF), 4 * sg_map->num_sg);
+                               DEBUG_MSG(KERN_INFO "riffa: fpga:%d chnl:%d, send sg buf populated, %d sent\n", sc->id, chnl, sg_map->num_sg);
+                       }
+                       break;
+
+               case EVENT_TXN_DONE:
+                       // Update with the true value of words transferred.
+                       sent = (((unsigned long long)msg)<<2);
+                       // Return as this is the end of the transaction.
+                       free_sg_buf(sc, sc->send[chnl]->sg_map_0);
+                       free_sg_buf(sc, sc->send[chnl]->sg_map_1);
+                       DEBUG_MSG(KERN_INFO "riffa: fpga:%d chnl:%d, sent %d words\n", sc->id, chnl, (unsigned int)(sent>>2));
+                       return (unsigned int)(sent>>2);
+                       break;
+
+               default:
+                       printk(KERN_ERR "riffa: fpga:%d chnl:%d, received unknown msg: %08x\n", sc->id, chnl, msg);
+                       break;
+               }
+       }
+
+       return 0;
+}
+
+static inline unsigned int chnl_send_wrapcheck(struct fpga_state * sc, int chnl,
+                               const char  __user * bufp, unsigned int len, unsigned int offset,
+                               unsigned int last, unsigned long long timeout)
+{
+       unsigned long long length = (((unsigned long long)len)<<2);
+       unsigned long udata = (unsigned long)bufp;
+       unsigned long max_ptr;
+       unsigned int ret;
+
+       // Validate the parameters.
+       if (chnl >= sc->num_chnls || chnl < 0) {
+               printk(KERN_INFO "riffa: fpga:%d chnl:%d, send channel invalid!\n", sc->id, chnl);
+               return 0;
+       }
+       max_ptr = (unsigned long)(udata + length - 1);
+       if (max_ptr < udata) {
+               printk(KERN_ERR "riffa: fpga:%d chnl:%d, send pointer address overflow\n", sc->id, chnl);
+               return -EINVAL;
+       }
+       if (udata & 0x3) {
+               printk(KERN_INFO "riffa: fpga:%d chnl:%d, send user buffer must be 32 bit word aligned!\n", sc->id, chnl);
+               return -EINVAL;
+       }
+
+       // Ensure no simultaneous operations from several threads
+       if (test_and_set_bit(CHNL_FLAG_BUSY, &sc->send[chnl]->flags) != 0) {
+               printk(KERN_ERR "riffa: fpga:%d chnl:%d, send conflict between threads!\n", sc->id, chnl);
+               return -EBUSY;
+       }
+
+       ret = chnl_send(sc, chnl, bufp, len, offset, last, timeout);
+
+       // Clear the busy flag
+       clear_bit(CHNL_FLAG_BUSY, &sc->send[chnl]->flags);
+
+       return ret;
+}
+
+/**
+ * Populates the fpga_info struct with the current FPGA state information. On
+ * success, returns 0. On error, returns a negative value.
+ */
+static inline int list_fpgas(fpga_info_list * list)
+{
+       int i;
+       int num_fpgas = 0;
+       struct fpga_state * sc;
+
+       for (i = 0; i < NUM_FPGAS; ++i) {
+               if (atomic_read(&used_fpgas[i])) {
+                       sc = fpgas[i];
+                       list->id[num_fpgas] = sc->id;
+                       list->num_chnls[num_fpgas] = sc->num_chnls;
+                       list->vendor_id[num_fpgas] = sc->vendor_id;
+                       list->device_id[num_fpgas] = sc->device_id;
+                       memcpy(list->name[num_fpgas], sc->name, 16*sizeof(char));
+                       num_fpgas++;
+               }
+       }
+       // Zero out the rest
+       for (i = num_fpgas; i < NUM_FPGAS; ++i) {
+               list->id[i] = -1;
+               list->num_chnls[i] = 0;
+               list->vendor_id[i] = 0;
+               list->device_id[i] = 0;
+               memset(list->name[i], 0, 16*sizeof(char));
+       }
+       list->num_fpgas = num_fpgas;
+
+       return 0;
+}
+
+/**
+ * Resets the driver for the specified FPGA. The fpga_state struct for all
+ * channels will be reset as will the FPGA itself.
+ */
+static inline void reset(int id)
+{
+       int i;
+       unsigned int dummy0;
+       unsigned int dummy1;
+       struct fpga_state * sc;
+
+       if (atomic_read(&used_fpgas[id])) {
+               sc = fpgas[id];
+               // Disable interrupts
+               atomic_set(&sc->intr_disabled, 1);
+               // Reset the FPGA
+               read_reg(sc, INFO_REG_OFF);
+
+               // Reset all the channels
+               for (i = 0; i < sc->num_chnls; ++i) {
+                       while (!pop_circ_queue(sc->send[i]->msgs, &dummy0, &dummy1));
+                       while (!pop_circ_queue(sc->recv[i]->msgs, &dummy0, &dummy1));
+                       wake_up(&sc->send[i]->waitq);
+                       wake_up(&sc->recv[i]->waitq);
+                       clear_bit(CHNL_FLAG_BUSY, &sc->send[i]->flags);
+                       clear_bit(CHNL_FLAG_BUSY, &sc->recv[i]->flags);
+               }
+               // Enable interrupts
+               atomic_set(&sc->intr_disabled, 0);
+       }
+}
+
+/**
+ * Main entry point for reading and writing on the device. Return value depends
+ * on ioctlnum and expected behavior. See code for details.
+ */
+static long fpga_ioctl(struct file *filp, unsigned int ioctlnum,
+               unsigned long ioctlparam)
+{
+       int rc;
+       fpga_chnl_io io;
+       fpga_info_list list;
+
+       switch (ioctlnum) {
+       case IOCTL_SEND:
+               if ((rc = copy_from_user(&io, (void *)ioctlparam, sizeof(fpga_chnl_io)))) {
+                       printk(KERN_ERR "riffa: cannot read ioctl user parameter.\n");
+                       return rc;
+               }
+               if (io.id < 0 || io.id >= NUM_FPGAS || !atomic_read(&used_fpgas[io.id]))
+                       return 0;
+               return chnl_send_wrapcheck(fpgas[io.id], io.chnl, io.data, io.len, io.offset,
+                               io.last, io.timeout);
+       case IOCTL_RECV:
+               if ((rc = copy_from_user(&io, (void *)ioctlparam, sizeof(fpga_chnl_io)))) {
+                       printk(KERN_ERR "riffa: cannot read ioctl user parameter.\n");
+                       return rc;
+               }
+               if (io.id < 0 || io.id >= NUM_FPGAS || !atomic_read(&used_fpgas[io.id]))
+                       return 0;
+               return chnl_recv_wrapcheck(fpgas[io.id], io.chnl, io.data, io.len, io.timeout);
+       case IOCTL_LIST:
+               list_fpgas(&list);
+               if ((rc = copy_to_user((void *)ioctlparam, &list, sizeof(fpga_info_list))))
+                       printk(KERN_ERR "riffa: cannot write ioctl user parameter.\n");
+               return rc;
+       case IOCTL_RESET:
+               reset((int)ioctlparam);
+               break;
+       default:
+               return -ENOTTY;
+               break;
+       }
+       return 0;
+}
+
+
+///////////////////////////////////////////////////////
+// PCI DRIVER HANDLERS
+///////////////////////////////////////////////////////
+
+/**
+ * Allocates and initializes chnl_dir structs for each channel. Returns the
+ * number of chnl_dir structs allocated.
+ */
+static inline int __devinit allocate_chnls(struct pci_dev *dev, struct fpga_state * sc)
+{
+       int i;
+       dma_addr_t hw_addr;
+
+       for (i = 0; i < sc->num_chnls; ++i) {
+               // Allocate the recv struct
+               sc->recv[i] = (struct chnl_dir *) kzalloc(sizeof(struct chnl_dir), GFP_KERNEL);
+               if (sc->recv[i] == NULL)
+                       return i;
+               sc->recv[i]->flags = 0;
+               init_waitqueue_head(&sc->recv[i]->waitq);
+               if ((sc->recv[i]->msgs = init_circ_queue(5)) == NULL) {
+                       kfree(sc->recv[i]);
+                       return i;
+               }
+               sc->recv[i]->buf_addr = pci_alloc_consistent(dev, sc->sg_buf_size, &hw_addr);
+               sc->recv[i]->buf_hw_addr = hw_addr;
+               if (sc->recv[i]->buf_addr == NULL) {
+                       free_circ_queue(sc->recv[i]->msgs);
+                       kfree(sc->recv[i]);
+                       return i;
+               }
+
+               // Allocate the send struct
+               sc->send[i] = (struct chnl_dir *) kzalloc(sizeof(struct chnl_dir), GFP_KERNEL);
+               if (sc->send[i] == NULL) {
+                       pci_free_consistent(dev, sc->sg_buf_size, sc->recv[i]->buf_addr,
+                                       (dma_addr_t)sc->recv[i]->buf_hw_addr);
+                       free_circ_queue(sc->recv[i]->msgs);
+                       kfree(sc->recv[i]);
+                       return i;
+               }
+               sc->send[i]->flags = 0;
+               init_waitqueue_head(&sc->send[i]->waitq);
+               if ((sc->send[i]->msgs = init_circ_queue(4)) == NULL) {
+                       kfree(sc->send[i]);
+                       pci_free_consistent(dev, sc->sg_buf_size, sc->recv[i]->buf_addr,
+                                       (dma_addr_t)sc->recv[i]->buf_hw_addr);
+                       free_circ_queue(sc->recv[i]->msgs);
+                       kfree(sc->recv[i]);
+                       return i;
+               }
+               sc->send[i]->buf_addr = pci_alloc_consistent(dev, sc->sg_buf_size, &hw_addr);
+               sc->send[i]->buf_hw_addr = hw_addr;
+               if (sc->send[i]->buf_addr == NULL) {
+                       free_circ_queue(sc->send[i]->msgs);
+                       kfree(sc->send[i]);
+                       pci_free_consistent(dev, sc->sg_buf_size, sc->recv[i]->buf_addr,
+                                       (dma_addr_t)sc->recv[i]->buf_hw_addr);
+                       free_circ_queue(sc->recv[i]->msgs);
+                       kfree(sc->recv[i]);
+                       return i;
+               }
+       }
+
+       return i;
+}
+
+/**
+ * Called by the OS when the device is ready for access. Returns 0 on success,
+ * negative value on failure.
+ */
+static int __devinit fpga_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+       int i;
+       int j;
+       int error;
+       struct fpga_state * sc;
+       dma_addr_t hw_addr;
+       unsigned int reg;
+       u32 lnkctl_result = 0;
+       u32 devctl_result = 0;
+       u32 devctl2_result = 0;
+
+       // Setup the PCIe device.
+       error = pci_enable_device(dev);
+       if (error < 0) {
+               printk(KERN_ERR "riffa: pci_enable_device returned %d\n", error);
+               return (-ENODEV);
+       }
+
+       // Enable bus master
+       pci_set_master(dev);
+
+       // Set the mask size
+       error = pci_set_dma_mask(dev, DMA_BIT_MASK(64));
+       if (!error)
+               error = pci_set_consistent_dma_mask(dev, DMA_BIT_MASK(64));
+       if (error) {
+               printk(KERN_ERR "riffa: cannot set 64 bit DMA mode\n");
+               pci_disable_device(dev);
+               return error;
+       }
+
+       // Allocate device structure.
+       sc = kzalloc(sizeof(*sc), GFP_KERNEL);
+       if (sc == NULL) {
+               printk(KERN_ERR "riffa: not enough memory to allocate sc\n");
+               pci_disable_device(dev);
+               return (-ENOMEM);
+       }
+       atomic_set(&sc->intr_disabled, 0);
+       snprintf(sc->name, sizeof(sc->name), "%s%d", pci_name(dev), 0);
+       sc->vendor_id = dev->vendor;
+       sc->device_id = dev->device;
+       printk(KERN_INFO "riffa: found FPGA with name: %s\n", sc->name);
+       printk(KERN_INFO "riffa: vendor id: 0x%04X\n", sc->vendor_id);
+       printk(KERN_INFO "riffa: device id: 0x%04X\n", sc->device_id);
+
+       // Setup the BAR memory regions
+       error = pci_request_regions(dev, sc->name);
+       if (error < 0) {
+               printk(KERN_ERR "riffa: pci_request_regions returned error: %d\n", error);
+               pci_disable_device(dev);
+               kfree(sc);
+               return (-ENODEV);
+       }
+
+       // PCI BAR 0
+       sc->bar0_addr = pci_resource_start(dev, 0);
+       sc->bar0_len = pci_resource_len(dev, 0);
+       sc->bar0_flags = pci_resource_flags(dev, 0);
+       printk(KERN_INFO "riffa: BAR 0 address: %llx\n", sc->bar0_addr);
+       printk(KERN_INFO "riffa: BAR 0 length: %lld bytes\n", sc->bar0_len);
+       if (sc->bar0_len != 1024) {
+               printk(KERN_ERR "riffa: BAR 0 incorrect length\n");
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return (-ENODEV);
+       }
+       sc->bar0 = ioremap(sc->bar0_addr, sc->bar0_len);
+       if (!sc->bar0) {
+               printk(KERN_ERR "riffa: could not ioremap BAR 0\n");
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return (-ENODEV);
+       }
+
+       // Setup MSI interrupts
+       error = pci_enable_msi(dev);
+       if (error != 0) {
+               printk(KERN_ERR "riffa: pci_enable_msi returned error: %d\n", error);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return error;
+       }
+
+       // Request an interrupt
+       error = request_irq(dev->irq, intrpt_handler, IRQF_SHARED, sc->name, sc);
+       if (error != 0) {
+               printk(KERN_ERR "riffa: request_irq(%d) returned error: %d\n", dev->irq, error);
+               pci_disable_msi(dev);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return error;
+       }
+       sc->irq = dev->irq;
+       printk(KERN_INFO "riffa: MSI setup on irq %d\n", dev->irq);
+
+       // Set extended tag bit
+    error = pcie_capability_read_dword(dev,PCI_EXP_DEVCTL,&devctl_result);
+       if (error != 0) {
+               printk(KERN_ERR "riffa: pcie_capability_read_dword returned error: %d\n", error);
+               free_irq(dev->irq, sc);
+               pci_disable_msi(dev);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return error;
+       }
+       printk(KERN_INFO "riffa: PCIE_EXP_DEVCTL register: %x\n",devctl_result);
+
+       error = pcie_capability_write_dword(dev,PCI_EXP_DEVCTL,(devctl_result|PCI_EXP_DEVCTL_EXT_TAG));
+       if (error != 0) {
+               printk(KERN_ERR "riffa: pcie_capability_write_dword returned error: %d\n", error);
+               free_irq(dev->irq, sc);
+               pci_disable_msi(dev);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return error;
+       }
+
+       // Set IDO bits
+       error = pcie_capability_read_dword(dev,PCI_EXP_DEVCTL2,&devctl2_result);
+       if (error != 0) {
+               printk(KERN_ERR "riffa: pcie_capability_read_dword returned error: %d\n", error);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL,devctl_result);
+               free_irq(dev->irq, sc);
+               pci_disable_msi(dev);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return error;
+       }
+       printk(KERN_INFO "riffa: PCIE_EXP_DEVCTL2 register: %x\n",devctl2_result);
+
+       error = pcie_capability_write_dword(dev,PCI_EXP_DEVCTL2,(devctl2_result | PCI_EXP_DEVCTL2_IDO_REQ_EN | PCI_EXP_DEVCTL2_IDO_CMP_EN));
+       if (error != 0) {
+               printk(KERN_ERR "riffa: pcie_capability_write_dword returned error: %d\n", error);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL,devctl_result);
+               free_irq(dev->irq, sc);
+               pci_disable_msi(dev);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return error;
+       }
+
+       // Set RCB to 128
+    error = pcie_capability_read_dword(dev,PCI_EXP_LNKCTL,&lnkctl_result);
+       if (error != 0) {
+               printk(KERN_ERR "riffa: pcie_capability_read_dword returned error: %d\n", error);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL2,devctl2_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL,devctl_result);
+               free_irq(dev->irq, sc);
+               pci_disable_msi(dev);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return error;
+       }
+       printk(KERN_INFO "riffa: PCIE_EXP_LNKCTL register: %x\n",lnkctl_result);
+
+       error = pcie_capability_write_dword(dev,PCI_EXP_LNKCTL,(lnkctl_result|PCI_EXP_LNKCTL_RCB));
+       if (error != 0) {
+               printk(KERN_ERR "riffa: pcie_capability_write_dword returned error: %d\n", error);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL2,devctl2_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL,devctl_result);
+               free_irq(dev->irq, sc);
+               pci_disable_msi(dev);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return error;
+       }
+       // Read device configuration
+       reg = read_reg(sc, INFO_REG_OFF);
+       sc->num_chnls = ((reg>>0) & 0xF);
+       sc->num_sg = SG_ELEMS*((reg>>19) & 0xF);
+       sc->sg_buf_size = SG_BUF_SIZE*((reg>>19) & 0xF);
+       printk(KERN_INFO "riffa: number of channels: %d\n", ((reg>>0) & 0xF));
+       printk(KERN_INFO "riffa: bus interface width: %d\n", ((reg>>19) & 0xF)<<5);
+       printk(KERN_INFO "riffa: bus master enabled: %d\n", ((reg>>4) & 0x1));
+       printk(KERN_INFO "riffa: negotiated link width: %d\n", ((reg>>5) & 0x3F));
+       printk(KERN_INFO "riffa: negotiated link rate: %d MTs\n", ((reg>>11) & 0x3)*2500);
+       printk(KERN_INFO "riffa: max downstream payload: %d bytes\n", 128<<((reg>>13) & 0x7) );
+       printk(KERN_INFO "riffa: max upstream payload: %d bytes\n", 128<<((reg>>16) & 0x7) );
+
+       if (((reg>>4) & 0x1) != 1) {
+               printk(KERN_ERR "riffa: bus master not enabled!\n");
+               pcie_capability_write_dword(dev,PCI_EXP_LNKCTL,lnkctl_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL2,devctl2_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL,devctl_result);
+               free_irq(dev->irq, sc);
+               pci_disable_msi(dev);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return (-ENODEV);
+       }
+
+       if (((reg>>5) & 0x3F) == 0 || ((reg>>11) & 0x3) == 0) {
+               printk(KERN_ERR "riffa: bad link parameters!\n");
+               pcie_capability_write_dword(dev,PCI_EXP_LNKCTL,lnkctl_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL2,devctl2_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL,devctl_result);
+               free_irq(dev->irq, sc);
+               pci_disable_msi(dev);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return (-ENODEV);
+       }
+
+       if ((reg & 0xF) == 0 || (reg & 0xF) > MAX_CHNLS) {
+               printk(KERN_ERR "riffa: bad number of channels!\n");
+               pcie_capability_write_dword(dev,PCI_EXP_LNKCTL,lnkctl_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL2,devctl2_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL,devctl_result);
+               free_irq(dev->irq, sc);
+               pci_disable_msi(dev);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return (-ENODEV);
+       }
+
+       if (((reg>>19) & 0xF) == 0 || ((reg>>19) & 0xF) > MAX_BUS_WIDTH_PARAM) {
+               printk(KERN_ERR "riffa: bad bus width!\n");
+               pcie_capability_write_dword(dev,PCI_EXP_LNKCTL,lnkctl_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL2,devctl2_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL,devctl_result);
+               free_irq(dev->irq, sc);
+               pci_disable_msi(dev);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return (-ENODEV);
+       }
+
+       // Create chnl_dir structs.
+       sc->recv = (struct chnl_dir **) kzalloc(sc->num_chnls*sizeof(struct chnl_dir*), GFP_KERNEL);
+       sc->send = (struct chnl_dir **) kzalloc(sc->num_chnls*sizeof(struct chnl_dir*), GFP_KERNEL);
+       if (sc->recv == NULL || sc->send == NULL) {
+               printk(KERN_ERR "riffa: not enough memory to allocate chnl_dir arrays\n");
+               if (sc->recv != NULL)
+                       kfree(sc->recv);
+               if (sc->send != NULL)
+                       kfree(sc->send);
+               pcie_capability_write_dword(dev,PCI_EXP_LNKCTL,lnkctl_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL2,devctl2_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL,devctl_result);
+               free_irq(dev->irq, sc);
+               pci_disable_msi(dev);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return (-ENOMEM);
+       }
+       j = allocate_chnls(dev, sc);
+       if (j < sc->num_chnls) {
+               sc->num_chnls = j;
+               printk(KERN_ERR "riffa: not enough memory to allocate chnl_dir structs\n");
+               for (i = 0; i < sc->num_chnls; ++i) {
+                       pci_free_consistent(dev, sc->sg_buf_size, sc->send[i]->buf_addr,
+                                       (dma_addr_t)sc->send[i]->buf_hw_addr);
+                       pci_free_consistent(dev, sc->sg_buf_size, sc->recv[i]->buf_addr,
+                                       (dma_addr_t)sc->recv[i]->buf_hw_addr);
+                       free_circ_queue(sc->send[i]->msgs);
+                       free_circ_queue(sc->recv[i]->msgs);
+                       kfree(sc->send[i]);
+                       kfree(sc->recv[i]);
+               }
+               kfree(sc->recv);
+               kfree(sc->send);
+               pcie_capability_write_dword(dev,PCI_EXP_LNKCTL,lnkctl_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL2,devctl2_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL,devctl_result);
+               free_irq(dev->irq, sc);
+               pci_disable_msi(dev);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return (-ENOMEM);
+       }
+
+       // Create spill buffer (for overflow on receive).
+       sc->spill_buf_addr = pci_alloc_consistent(dev, SPILL_BUF_SIZE, &hw_addr);
+       sc->spill_buf_hw_addr = hw_addr;
+       if (sc->spill_buf_addr == NULL) {
+               printk(KERN_ERR "riffa: not enough memory to allocate spill buffer\n");
+               for (i = 0; i < sc->num_chnls; ++i) {
+                       pci_free_consistent(dev, sc->sg_buf_size, sc->send[i]->buf_addr,
+                                       (dma_addr_t)sc->send[i]->buf_hw_addr);
+                       pci_free_consistent(dev, sc->sg_buf_size, sc->recv[i]->buf_addr,
+                                       (dma_addr_t)sc->recv[i]->buf_hw_addr);
+                       free_circ_queue(sc->send[i]->msgs);
+                       free_circ_queue(sc->recv[i]->msgs);
+                       kfree(sc->send[i]);
+                       kfree(sc->recv[i]);
+               }
+               pcie_capability_write_dword(dev,PCI_EXP_LNKCTL,lnkctl_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL2,devctl2_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL,devctl_result);
+               kfree(sc->recv);
+               kfree(sc->send);
+               free_irq(dev->irq, sc);
+               pci_disable_msi(dev);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+       }
+
+       // Save pointer to structure
+       pci_set_drvdata(dev, sc);
+       sc->dev = dev;
+       sc->id = -1;
+       for (i = 0; i < NUM_FPGAS; i++) {
+               if (!atomic_xchg(&used_fpgas[i], 1)) {
+                       sc->id = i;
+                       fpgas[i] = sc;
+                       break;
+               }
+       }
+       if (sc->id == -1) {
+               printk(KERN_ERR "riffa: could not save FPGA information, %d is limit.\n", NUM_FPGAS);
+               for (i = 0; i < sc->num_chnls; ++i) {
+                       pci_free_consistent(dev, sc->sg_buf_size, sc->send[i]->buf_addr,
+                                       (dma_addr_t)sc->send[i]->buf_hw_addr);
+                       pci_free_consistent(dev, sc->sg_buf_size, sc->recv[i]->buf_addr,
+                                       (dma_addr_t)sc->recv[i]->buf_hw_addr);
+                       free_circ_queue(sc->send[i]->msgs);
+                       free_circ_queue(sc->recv[i]->msgs);
+                       kfree(sc->send[i]);
+                       kfree(sc->recv[i]);
+               }
+               kfree(sc->recv);
+               kfree(sc->send);
+               pci_free_consistent(dev, SPILL_BUF_SIZE, sc->spill_buf_addr,
+                               (dma_addr_t)sc->spill_buf_hw_addr);
+               pcie_capability_write_dword(dev,PCI_EXP_LNKCTL,lnkctl_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL2,devctl2_result);
+               pcie_capability_write_dword(dev,PCI_EXP_DEVCTL,devctl_result);
+               free_irq(dev->irq, sc);
+               pci_disable_msi(dev);
+               iounmap(sc->bar0);
+               pci_release_regions(dev);
+               pci_disable_device(dev);
+               kfree(sc);
+               return (-ENOMEM);
+       }
+       else {
+               printk(KERN_INFO "riffa: saved FPGA with id: %d\n", sc->id);
+       }
+
+       return 0;
+}
+
+/**
+ * Called when the device is unloaded.
+ */
+static void __devexit fpga_remove(struct pci_dev *dev)
+{
+       int i;
+       u32 result;
+       struct fpga_state * sc;
+
+       pcie_capability_read_dword(dev,PCI_EXP_DEVCTL,&result);
+       pcie_capability_write_dword(dev,PCI_EXP_DEVCTL,result & ~(PCI_EXP_DEVCTL_EXT_TAG | PCI_EXP_DEVCTL_RELAX_EN));
+
+       pcie_capability_read_dword(dev,PCI_EXP_DEVCTL2,&result);
+       pcie_capability_write_dword(dev,PCI_EXP_DEVCTL2,result & ~(PCI_EXP_DEVCTL2_IDO_REQ_EN | PCI_EXP_DEVCTL2_IDO_CMP_EN));
+
+       pcie_capability_read_dword(dev,PCI_EXP_DEVCTL,&result);
+       pcie_capability_write_dword(dev,PCI_EXP_LNKCTL,result & (~PCI_EXP_LNKCTL_RCB));
+
+       if ((sc = (struct fpga_state *)pci_get_drvdata(dev)) != NULL) {
+               // Free structs, memory regions, etc.
+               atomic_set(&used_fpgas[sc->id], 0);
+               for (i = 0; i < sc->num_chnls; ++i) {
+                       pci_free_consistent(dev, sc->sg_buf_size, sc->send[i]->buf_addr,
+                                       (dma_addr_t)sc->send[i]->buf_hw_addr);
+                       pci_free_consistent(dev, sc->sg_buf_size, sc->recv[i]->buf_addr,
+                                       (dma_addr_t)sc->recv[i]->buf_hw_addr);
+                       free_circ_queue(sc->send[i]->msgs);
+                       free_circ_queue(sc->recv[i]->msgs);
+                       kfree(sc->send[i]);
+                       kfree(sc->recv[i]);
+               }
+               kfree(sc->recv);
+               kfree(sc->send);
+               pci_free_consistent(dev, SPILL_BUF_SIZE, sc->spill_buf_addr,
+                               (dma_addr_t)sc->spill_buf_hw_addr);
+               free_irq(dev->irq, sc);
+               iounmap(sc->bar0);
+               kfree(sc);
+       }
+       pci_disable_msi(dev);
+       pci_release_regions(dev);
+       pci_disable_device(dev);
+       pci_set_drvdata(dev, NULL);
+}
+
+
+///////////////////////////////////////////////////////
+// MODULE INIT/EXIT FUNCTIONS
+///////////////////////////////////////////////////////
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4,8,0)
+static DEFINE_PCI_DEVICE_TABLE(fpga_ids) =
+#else
+static const struct pci_device_id fpga_ids[] =
+#endif
+{
+       {PCI_DEVICE(VENDOR_ID0, PCI_ANY_ID)},
+       {PCI_DEVICE(VENDOR_ID1, PCI_ANY_ID)},
+       {0},
+};
+
+MODULE_DEVICE_TABLE(pci, fpga_ids);
+static struct pci_driver fpga_driver = {
+       .name           = DEVICE_NAME,
+       .id_table       = fpga_ids,
+       .probe          = fpga_probe,
+       .remove         = __devexit_p(fpga_remove),
+};
+
+static const struct file_operations fpga_fops = {
+       .owner                  = THIS_MODULE,
+       .unlocked_ioctl = fpga_ioctl,
+};
+
+/**
+ * Called to initialize the PCI device.
+ */
+static int __init fpga_init(void)
+{
+       int i;
+       int error;
+
+       for (i = 0; i < NUM_FPGAS; i++)
+               atomic_set(&used_fpgas[i], 0);
+
+       error = pci_register_driver(&fpga_driver);
+       if (error != 0) {
+               printk(KERN_ERR "riffa: pci_module_register returned %d\n", error);
+               return (error);
+       }
+
+       error = register_chrdev(MAJOR_NUM, DEVICE_NAME, &fpga_fops);
+       if (error < 0) {
+               printk(KERN_ERR "riffa: register_chrdev returned %d\n", error);
+               return (error);
+       }
+
+       mymodule_class = class_create(THIS_MODULE, DEVICE_NAME);
+       if (IS_ERR(mymodule_class)) {
+               error = PTR_ERR(mymodule_class);
+               printk(KERN_ERR "riffa: class_create() returned %d\n", error);
+               return (error);
+       }
+
+       devt = MKDEV(MAJOR_NUM, 0);
+       device_create(mymodule_class, NULL, devt, "%s", DEVICE_NAME);
+
+       return 0;
+}
+
+/**
+ * Called to destroy the PCI device.
+ */
+static void __exit fpga_exit(void)
+{
+       device_destroy(mymodule_class, devt);
+       class_destroy(mymodule_class);
+       pci_unregister_driver(&fpga_driver);
+       unregister_chrdev(MAJOR_NUM, DEVICE_NAME);
+}
+
+module_init(fpga_init);
+module_exit(fpga_exit);
diff --git a/drivers/staging/riffa/riffa_driver.h b/drivers/staging/riffa/riffa_driver.h
new file mode 100644
index 000000000000..cf366b269c18
--- /dev/null
+++ b/drivers/staging/riffa/riffa_driver.h
@@ -0,0 +1,131 @@
+// ----------------------------------------------------------------------
+// Copyright (c) 2016, The Regents of the University of California All
+// rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//
+//     * Redistributions in binary form must reproduce the above
+//       copyright notice, this list of conditions and the following
+//       disclaimer in the documentation and/or other materials provided
+//       with the distribution.
+//
+//     * Neither the name of The Regents of the University of California
+//       nor the names of its contributors may be used to endorse or
+//       promote products derived from this software without specific
+//       prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL REGENTS OF THE
+// UNIVERSITY OF CALIFORNIA BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+// TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+// DAMAGE.
+// ----------------------------------------------------------------------
+
+/*
+ * Filename: riffa_driver.h
+ * Version: 2.0
+ * Description: Linux PCIe device driver for RIFFA. Uses Linux kernel APIs in
+ *  version 2.6.27+ (tested on version 2.6.32 - 3.3.0).
+ * Author: Matthew Jacobsen
+ * History: @mattj: Initial release. Version 2.0.
+ */
+
+#ifndef RIFFA_DRIVER_H
+#define RIFFA_DRIVER_H
+
+#include <linux/ioctl.h>
+
+#define DEBUG 1
+
+#ifdef DEBUG
+#define DEBUG_MSG(...) printk(__VA_ARGS__)
+#else
+#define DEBUG_MSG(...)
+#endif
+
+
+// The major device number. We can't rely on dynamic registration because ioctls
+// need to know it.
+#define MAJOR_NUM 100
+#define DEVICE_NAME "riffa"
+#define VENDOR_ID0 0x10EE
+#define VENDOR_ID1 0x1172
+
+// Message events for readmsgs/writemsgs queues.
+#define EVENT_TXN_LEN                          1
+#define EVENT_TXN_OFFLAST                      2
+#define EVENT_TXN_DONE                         3
+#define EVENT_SG_BUF_READ                      4
+
+// Constants and device offsets
+#define NUM_FPGAS                                      5       // max # of FPGAs to support in a single PC
+#define MAX_CHNLS                                      12      // max # of channels per FPGA
+#define MAX_BUS_WIDTH_PARAM                    4       // max bus width parameter
+#define SG_BUF_SIZE                                    (4*1024)        // size of shared SG buffer
+#define SG_ELEMS                                       200 // # of SG elements to transfer at a time
+#define SPILL_BUF_SIZE                         (4*1024)        // size of shared spill common buffer
+
+#define RX_SG_LEN_REG_OFF                      0x0     // config offset for RX SG buf length
+#define RX_SG_ADDR_LO_REG_OFF          0x1     // config offset for RX SG buf low addr
+#define RX_SG_ADDR_HI_REG_OFF          0x2     // config offset for RX SG buf high addr
+#define RX_LEN_REG_OFF                         0x3     // config offset for RX txn length
+#define RX_OFFLAST_REG_OFF                     0x4     // config offset for RX txn last/offset
+#define RX_TNFR_LEN_REG_OFF                    0xD     // config offset for RX transfer length
+#define TX_SG_LEN_REG_OFF                      0x5     // config offset for TX SG buf length
+#define TX_SG_ADDR_LO_REG_OFF          0x6     // config offset for TX SG buf low addr
+#define TX_SG_ADDR_HI_REG_OFF          0x7     // config offset for TX SG buf high addr
+#define TX_LEN_REG_OFF                         0x8     // config offset for TX txn length
+#define TX_OFFLAST_REG_OFF                     0x9     // config offset for TX txn last/offset
+#define TX_TNFR_LEN_REG_OFF                    0xE     // config offset for TX transfer length
+
+#define INFO_REG_OFF                           0xA     // config offset for link info
+
+#define IRQ_REG0_OFF                           0xB     // config offset for interrupt reg 0
+#define IRQ_REG1_OFF                           0xC     // config offset for interrupt reg 1
+
+
+// Structs
+struct fpga_chnl_io
+{
+       int id;
+       int chnl;
+       unsigned int len;
+       unsigned int offset;
+       unsigned int last;
+       unsigned long long timeout;
+       char * data;
+};
+typedef struct fpga_chnl_io fpga_chnl_io;
+
+struct fpga_info_list
+{
+       int num_fpgas;
+       int id[NUM_FPGAS];
+       int num_chnls[NUM_FPGAS];
+       char name[NUM_FPGAS][16];
+       int vendor_id[NUM_FPGAS];
+       int device_id[NUM_FPGAS];
+};
+typedef struct fpga_info_list fpga_info_list;
+
+// IOCTLs
+#define IOCTL_SEND _IOW(MAJOR_NUM, 1, fpga_chnl_io *)
+#define IOCTL_RECV _IOR(MAJOR_NUM, 2, fpga_chnl_io *)
+#define IOCTL_LIST _IOR(MAJOR_NUM, 3, fpga_info_list *)
+#define IOCTL_RESET _IOW(MAJOR_NUM, 4, int)
+
+
+
+#endif
--
2.17.1

---
_______________________________________________
devel mailing list
devel@xxxxxxxxxxxxxxxxxxxxxx
http://driverdev.linuxdriverproject.org/mailman/listinfo/driverdev-devel





[Index of Archives]     [Linux Driver Backports]     [DMA Engine]     [Linux GPIO]     [Linux SPI]     [Video for Linux]     [Linux USB Devel]     [Linux Coverity]     [Linux Audio Users]     [Linux Kernel]     [Linux SCSI]     [Yosemite Backpacking]
  Powered by Linux