diff options
Diffstat (limited to 'sources')
110 files changed, 24515 insertions, 0 deletions
diff --git a/sources/api/.gitignore b/sources/api/.gitignore new file mode 100644 index 0000000..be60ba4 --- /dev/null +++ b/sources/api/.gitignore @@ -0,0 +1,2 @@ +*.o +.vscode/
\ No newline at end of file diff --git a/sources/api/board.cpp b/sources/api/board.cpp new file mode 100644 index 0000000..db990ca --- /dev/null +++ b/sources/api/board.cpp @@ -0,0 +1,132 @@ +#include "board.h" +#include <unistd.h> +#include <fstream> +#include <iostream> +#include <cassert> +#include <fcntl.h> +#include <string.h> + +BoardInterface::BoardInterface(IFACE iface_type) +{ + this -> iface_type = iface_type; +} +BoardInterface::~BoardInterface() +{ + free(send_buf); + free(recv_buf); + close(to_card); + close(from_card); +} + +int BoardInterface::init() +{ + switch(iface_type) + { + case IFACE::XDMA: + { + int fpga_fd = open(TO_FPGA_DEFAULT.c_str(), O_RDWR); + if(fpga_fd<0) + { + std::cerr << "Open to card failed!" << std::endl; + return 1; + } + else + std::cout << "Opened " << TO_FPGA_DEFAULT << " -> " << fpga_fd << std::endl; + to_card = fpga_fd; + fpga_fd = open(FROM_FPGA_DEFAULT.c_str(), O_RDWR); + if(fpga_fd<0) + { + std::cerr << "Open to host failed!" << std::endl; + return 1; + } + else + std::cout << "Opened " << FROM_FPGA_DEFAULT << " -> " << fpga_fd << std::endl; + from_card = fpga_fd; + // allocate page size aligned X page size regions to our buffers + if (posix_memalign((void **)&send_buf, 4096 /*alignment */ , SEND_BUF_SIZE + (4096-(SEND_BUF_SIZE % 4096))) != 0) + { + std::cerr << "Send buffer allocation failed!" << std::endl; + return 1; + } + if (posix_memalign((void **)&recv_buf, 4096 /*alignment */ , RECV_BUF_SIZE + (4096-(RECV_BUF_SIZE % 4096)))) + { + std::cerr << "Receive buffer allocation failed!" << std::endl; + return 1; + } + if( (!send_buf) || (!recv_buf) ) + { + std::cerr << "Buffers cannot be allocated!" << std::endl; + return 1; + } + return 0; + } + default: + std::cerr << "Unknown iface_type!" << std::endl; + return 1; + } +} + +int BoardInterface::sendData(void* data, const uint size) +{ + switch(iface_type) + { + case IFACE::XDMA: + return xdma_send(data,size); + break; + default: + std::cerr << "Unknown iface_type!" << std::endl; + return 1; + } +} + +int BoardInterface::recvData(void* buf, const uint size) +{ + switch(iface_type) + { + case IFACE::XDMA: + return xdma_recv(buf,size); + break; + default: + std::cerr << "Unknown iface_type!" << std::endl; + return 1; + } +} + +int BoardInterface::xdma_send(void* data, const uint size) +{ + memcpy((char*)send_buf, (char*)data, size); + + int fd = to_card; + ssize_t rc; + uint64_t count = 0; + char *buf = (char*) send_buf; + + while (count < size) { + /* write data to file from memory buffer */ + rc = write(fd, buf, size); + assert(rc == size || rc == 0); + count += rc; + } + + // We wrote more than we supposed to + if (count != size) + return 1; + + return 0; +} + +int BoardInterface::xdma_recv(void* buf, const uint size) +{ + assert(size <= RECV_BUF_SIZE && "given read size is too large"); + + int fd = from_card; + uint64_t count = 0; + // Try to read from the card. + count = read(fd, (char*) recv_buf, size); + + // We read more than we were supposed to + assert(count <= size);// || rc == 0); + + memcpy(buf, (char*) recv_buf, count); + return count; +} diff --git a/sources/api/board.h b/sources/api/board.h new file mode 100644 index 0000000..0935392 --- /dev/null +++ b/sources/api/board.h @@ -0,0 +1,35 @@ +#include <string> + +#ifndef BOARD_H +#define BOARD_H +/** This class defines how the host + * interfaces with the board. + */ +class BoardInterface{ + const uint SEND_BUF_SIZE = 32*2048; + // send each instruction as a 256-bit packet + const uint RECV_BUF_SIZE = 1024*32; + const std::string TO_FPGA_DEFAULT = "/dev/xdma0_h2c_0"; + const std::string FROM_FPGA_DEFAULT = "/dev/xdma0_c2h_0"; +public: + enum class IFACE { + XDMA = 0 + }; + BoardInterface(IFACE); + ~BoardInterface(); + int init(); + int sendData(void* data, const uint size); + int recvData(void* buf , const uint size); +private: + IFACE iface_type; + // XDMA related constructs + int to_card; + int from_card; + void* send_buf; + void* recv_buf; + int xdma_send(void* data, const uint size); + int xdma_recv(void* buf, const uint size); + // end XDMA related constructs +}; + +#endif diff --git a/sources/api/instruction.cpp b/sources/api/instruction.cpp new file mode 100644 index 0000000..432188f --- /dev/null +++ b/sources/api/instruction.cpp @@ -0,0 +1,586 @@ +#include "instruction.h" +#include <assert.h> +#include <stdio.h> + +Inst SMC_ADD(int rs1, int rs2, int rt) +{ + Inst fu_code = (uint64_t)__ADD << __FU_CODE; + Inst s_regs = (rs2 << __RS2) | rs1; + Inst t_reg = rt << __RT; + + Inst inst = fu_code | s_regs | t_reg; + + return inst; +} +Inst SMC_ADDI(int rs1, uint32_t imd, int rt) +{ + Inst fu_code = (uint64_t)__ADDI << __FU_CODE; + Inst s_reg = rs1; + Inst imd1 = imd << __IMD1; + Inst t_reg = rt << __RT; + + Inst inst = fu_code | s_reg | imd1 | t_reg; + + return inst; +} +Inst SMC_SUB(int rs1, int rs2, int rt) +{ + Inst fu_code = (uint64_t)__SUB << __FU_CODE; + Inst s_regs = (rs2 << __RS2) | rs1; + Inst t_reg = rt << __RT; + + Inst inst = fu_code | s_regs | t_reg; + + return inst; +} +Inst SMC_SUBI(int rs1, uint32_t imd, int rt) +{ + Inst fu_code = (uint64_t)__SUBI << __FU_CODE; + Inst s_reg = rs1; + Inst imd1 = imd << __IMD1; + Inst t_reg = rt << __RT; + + Inst inst = fu_code | s_reg | imd1 | t_reg; + + return inst; +} +Inst SMC_LI(uint32_t imd, int rt) +{ + Inst fu_code = (uint64_t)__LI << __FU_CODE; + Inst imd1 = ((uint32_t)(imd<<16)>>16) << __IMD1; + Inst imd2 = (uint64_t)(((uint32_t)imd)>>16) << __IMD3; + Inst t_reg = rt << __RT; + + Inst inst = fu_code | imd1 | imd2 | t_reg; + + return inst; +} +Inst SMC_MV(int rs1, int rt) +{ + Inst fu_code = (uint64_t)__MV << __FU_CODE; + Inst s_reg = rs1; + Inst t_reg = rt << __RT; + + Inst inst = fu_code | s_reg | t_reg; + + return inst; +} +Inst SMC_SRC(int rs1, int rt) +{ + Inst fu_code = (uint64_t)__SRC << __FU_CODE; + Inst s_reg = rs1; + Inst t_reg = rt << __RT; + + Inst inst = fu_code | s_reg | t_reg; + + return inst; +} +Inst SMC_LDWD(int rs1, int off) +{ + Inst fu_code = (uint64_t)__LDWD << __FU_CODE; + Inst s_reg = rs1; + Inst offset = off << __RT; + + Inst inst = fu_code | s_reg | offset; + + return inst; +} +Inst SMC_LDPC(PC_TYPE pc_type, int rt) +{ + Inst fu_code = (uint64_t)__LDPC << __FU_CODE; + Inst pc_reg = 2; + Inst t_reg = rt << __RT; + switch(pc_type){ + case PC_TYPE::WRITE: + pc_reg = 0; + break; + case PC_TYPE::READ: + pc_reg = 1; + break; + case PC_TYPE::PRE: + pc_reg = 2; + break; + case PC_TYPE::ACT: + pc_reg = 3; + break; + case PC_TYPE::ZQ: + pc_reg = 4; + break; + case PC_TYPE::REF: + pc_reg = 5; + break; + case PC_TYPE::CYC: + pc_reg = 6; + break; + } + + Inst inst = fu_code | pc_reg | t_reg; + + return inst; +} +Inst SMC_BL(int rs1, int rs2, int tgt) +{ + Inst op_code = (uint64_t)0x1 << __IS_BR; + Inst fu_code = (uint64_t)__BL << __FU_CODE; + Inst s_regs = (rs2 << __RS2) | rs1; + Inst target = (uint64_t)tgt << __BR_TGT; + + Inst inst = op_code | fu_code | s_regs | target; + + return inst; +} +Inst SMC_BEQ(int rs1, int rs2, int tgt) +{ + Inst op_code = (uint64_t)0x1 << __IS_BR; + Inst fu_code = (uint64_t)__BEQ << __FU_CODE; + Inst s_regs = (rs2 << __RS2) | rs1; + Inst target = (uint64_t)tgt << __BR_TGT; + + Inst inst = op_code | fu_code | s_regs | target; + + return inst; +} +Inst SMC_JUMP(int tgt) +{ + Inst op_code = (uint64_t)0x1 << __IS_BR; + Inst fu_code = (uint64_t)__JUMP << __FU_CODE; + Inst target = tgt; + + Inst inst = op_code | fu_code | target; + + return inst; +} +Inst SMC_SLEEP(uint32_t samt) +{ + assert(samt > 2 && "Cannot sleep for less than 3 cycles."); + Inst op_code = (uint64_t)0x1 << __IS_BR; + Inst fu_code = (uint64_t)__SLEEP << __FU_CODE; + + samt -= 2; + + Inst inst = op_code | fu_code | samt; + + return inst; +} +Inst SMC_LD(int rb, int offset, int rt) +{ + Inst op_code = (uint64_t)0x1 << __IS_MEM; + Inst fu_code = (uint64_t)__LD << __FU_CODE; + Inst s_reg = rb; + Inst imd1 = offset << __IMD1; + Inst t_reg = rt << __RT; + + Inst inst = op_code | fu_code | s_reg | imd1 | t_reg; + + return inst; +} +Inst SMC_ST(int rb, int offset, int rv) +{ + Inst op_code = (uint64_t)0x1 << __IS_MEM; + Inst fu_code = (uint64_t)__ST << __FU_CODE; + Inst b_reg = rb; + Inst imd1 = offset << __IMD1; + Inst v_reg = rv << __RT; // We cannot have imd1 and rs2 present simultaneously + + Inst inst = op_code | fu_code | b_reg | imd1 | v_reg; + + return inst; +} +Inst SMC_AND(int rs1, int rs2, int rt) +{ + Inst op_code = (uint64_t)0x1 << __IS_BW; + Inst fu_code = (uint64_t)__AND << __FU_CODE; + Inst s_regs = (rs2 << __RS2) | rs1; + Inst t_reg = rt << __RT; + + Inst inst = op_code | fu_code | s_regs | t_reg; + + return inst; +} +Inst SMC_OR(int rs1, int rs2, int rt) +{ + Inst op_code = (uint64_t)0x1 << __IS_BW; + Inst fu_code = (uint64_t)__OR << __FU_CODE; + Inst s_regs = (rs2 << __RS2) | rs1; + Inst t_reg = rt << __RT; + + Inst inst = op_code | fu_code | s_regs | t_reg; + + return inst; +} +Inst SMC_XOR(int rs1, int rs2, int rt) +{ + Inst op_code = (uint64_t)0x1 << __IS_BW; + Inst fu_code = (uint64_t)__XOR << __FU_CODE; + Inst s_regs = (rs2 << __RS2) | rs1; + Inst t_reg = rt << __RT; + + Inst inst = op_code | fu_code | s_regs | t_reg; + + return inst; +} +Inst SMC_END() +{ + return 0; +} +Inst SMC_INFO(int rdcnt) +{ + Inst op_code = (uint64_t)0x1 << __IS_MISC; + Inst fu_code = (uint64_t)__INFO << __FU_CODE; + return op_code | fu_code | (uint64_t) rdcnt; +} +Mininst SMC_WRITE(int bar, int ibar, int car, int icar, int BL4, int ap) +{ + Mininst fu_code = ((uint64_t)__WRITE) << __DDR_CMD; + Mininst i_bar = bar; + Mininst i_car = car << __DDR_CAR; + Mininst i_ibar = ibar << __DDR_IBAR; + Mininst i_icar = icar << __DDR_ICAR; + Mininst i_BL4 = BL4 << __DDR_BL4; + Mininst i_ap = ap <<__DDR_AP; + + Mininst inst = fu_code | i_bar | i_car | i_ibar | + i_icar | i_BL4 | i_ap; + + return inst; +} +Mininst SMC_READ(int bar, int ibar, int car, int icar, int BL4, int ap) +{ + Mininst fu_code = ((uint64_t)__READ) << __DDR_CMD; + Mininst i_bar = bar; + Mininst i_car = car << __DDR_CAR; + Mininst i_ibar = ibar << __DDR_IBAR; + Mininst i_icar = icar << __DDR_ICAR; + Mininst i_BL4 = BL4 << __DDR_BL4; + Mininst i_ap = ap <<__DDR_AP; + + Mininst inst = fu_code | i_bar | i_car | i_ibar | + i_icar | i_BL4 | i_ap; + + return inst; +} +Mininst SMC_PRE(int bar, int ibar, int pall) +{ + Mininst fu_code = ((uint64_t)__PRE) << __DDR_CMD; + Mininst i_bar = bar; + Mininst i_ibar = ibar << __DDR_IBAR; + Mininst i_pall = pall << __DDR_PALL; + + Mininst inst = fu_code | i_bar | i_ibar | i_pall; + + return inst; +} +Mininst SMC_ACT(int bar, int ibar, int rar, int irar) +{ + Mininst fu_code = ((uint64_t)__ACT) << __DDR_CMD; + Mininst i_bar = bar; + Mininst i_rar = rar << __DDR_RAR; + Mininst i_ibar = ibar << __DDR_IBAR; + Mininst i_irar = irar << __DDR_IRAR; + + Mininst inst = fu_code | i_bar | i_rar | i_ibar | i_irar; + + return inst; +} +Mininst SMC_ZQ() +{ + Mininst fu_code = ((uint64_t)__ZQ) << __DDR_CMD; + + return fu_code; +} +Mininst SMC_REF() +{ + Mininst fu_code = ((uint64_t)__REF) << __DDR_CMD; + + return fu_code; +} +Mininst SMC_NOP() +{ + Mininst fu_code = ((uint64_t)__NOP) << __DDR_CMD; + + return fu_code; +} +Inst SMC_SRE() +{ + Inst op_code = (uint64_t) 0x1 << 56; + Inst fu_code = (uint64_t)__SRE << __FU_CODE; + + return fu_code | op_code; +} +Inst SMC_SRX() +{ + Inst op_code = (uint64_t) 0x1 << 56; + Inst fu_code = (uint64_t)__SRX << __FU_CODE; + + return fu_code | op_code; +} +Inst __pack_mininsts(Mininst i1, Mininst i2, Mininst i3, Mininst i4) +{ + return (uint64_t) i4 << 48 | + (uint64_t) i3 << 32 | + (uint64_t) i2 << 16 | + i1 ; +} + +int is_conditional(Inst i) +{ + uint64_t fcode = (i >> __FU_CODE) & 0x7ff; + switch (fcode) { + case 0: + return 1; + case 1: + return 1; + } + return 0; +} + +int is_branch(Inst i) +{ + uint64_t fcode = (i >> __OP_CODE); + //printf("%ld\n",fcode); + switch (fcode) { + case 8: + return 1; + } + return 0; +} + +int is_ddr(Inst i) +{ + uint64_t fcode = i >> __IS_DDR; + return fcode == 1; +} + +int is_load(Inst i) +{ + uint64_t fcode = i >> __FU_CODE; + return fcode == 4096; +} + +int is_sleep(Inst i) +{ + uint64_t fcode = i >> __FU_CODE; + return fcode == 19; +} + +int is_ddr_read(Inst inst) +{ + int ctr = 0; + for (size_t i = 0 ; i < 4 ; i++) + { + Mininst min = inst >> (i*16); + ctr += (min >> __DDR_CMD) == (Mininst) __READ; + } + return ctr; +} + +void decode_inst(Inst inst) +{ + if(is_ddr(inst)) + { + for(int i = 0 ; i < 4 ; i++) + { + Mininst mini = (inst >> i*16); + decode_ddr(mini); + if (i<3) printf(" : "); + } + } + else + { + int fu_code = inst >> __FU_CODE; + int fc_mask = 0x7ff; + fu_code = fu_code & fc_mask; + int op_code = inst >> __OP_CODE; + int is_br = op_code == 0x8; + int is_mem = op_code == 0x2; + int is_bw = op_code == 0x1; + int is_misc = op_code == 0x4; + int is_arit = op_code == 0x0; + uint64_t rid_mask = 0xf; + uint64_t imd_mask = 0xffff; + uint64_t br_tgt_mask = 0x7ffff; + uint64_t jmp_tgt_mask = 0x7ffffff; + int rs1 = (inst >> __RS1) & rid_mask; + int rs2 = (inst >> __RS2) & rid_mask; + int rt = (inst >> __RT) & rid_mask; + int imd1 = (inst >> __IMD1) & imd_mask; + int imd3 = (inst >> __IMD3) & imd_mask; + int bt = (inst >> __BR_TGT) & br_tgt_mask; + int jt = (inst >> __J_TGT) & jmp_tgt_mask; + int samt = (uint32_t)inst; + uint64_t imd_concat = ((uint64_t)imd3 << 16) + imd1; + if (is_arit) + { + switch (fu_code) + { + case __ADD: + if(inst == 0) + printf("END"); + else + printf("ADD r%d r%d r%d", rt, rs1, rs2); + break; + case __ADDI: + printf("ADDI r%d r%d %d", rt, rs1, imd1); + break; + case __SUB: + printf("SUB r%d r%d r%d", rt, rs1, rs2); + break; + case __SUBI: + printf("SUBI r%d r%d %d", rt, rs1, imd1); + break; + case __MV: + printf("MOV r%d r%d", rt, rs1); + break; + case __LI: + printf("LI r%d %ld", rt, imd_concat); + break; + case __SRC: + printf("SRC r%d r%d", rt, rs1); + break; + case __LDWD: + printf("LDWD %d r%d", rt, rs1); + break; + case __LDPC: + switch(rs1){ + case 0: + printf("LDPC r%d WRITE_COUNTER", rt); + break; + case 1: + printf("LDPC r%d READ_COUNTER", rt); + break; + case 2: + printf("LDPC r%d PRE_COUNTER", rt); + break; + case 3: + printf("LDPC r%d ACT_COUNTER", rt); + break; + case 4: + printf("LDPC r%d ZQ_COUNTER", rt); + break; + case 5: + printf("LDPC r%d REF_COUNTER", rt); + break; + case 6: + printf("LDPC r%d TOTAL_CYCLE", rt); + break; + } + break; + case __SRE: + printf("SRE"); + break; + case __SRX: + printf("SRX"); + break; + } + } else if (is_br) + { + switch (fu_code) + { + case __BL: + printf("BL PC:%d r%d r%d", bt, rs1, rs2); + break; + case __BEQ: + printf("BEQ PC:0x%x r%d r%d", bt, rs1, rs2); + break; + case __JUMP: + printf("JUMP PC:%d", jt); + break; + case __SLEEP: + printf("SLEEP %d cycles", samt); + break; + } + } else if (is_misc) + { + switch (fu_code) + { + case __INFO: + printf("Auto-generated instruction."); + break; + } + } else if (is_mem) + { + switch (fu_code) + { + case __LD: + printf("LD r%d [r%d]%d", rt, rs1, imd1); + break; + case __ST: + printf("ST [r%d]%d r%d", rs1, imd1, rt); + break; + } + } else if (is_bw) + { + switch (fu_code) + { + case __AND: + printf("AND r%d r%d r%d", rt, rs1, rs2); + break; + case __OR: + printf("OR r%d r%d r%d", rt, rs1, rs2); + break; + case __XOR: + printf("XOR r%d r%d r%d", rt, rs1, rs2); + break; + } + } + } +} + +void decode_ddr(Mininst i) +{ + int ddr_code = i >> __DDR_CMD; + uint64_t rid_mask = 0xf; + int car = (i >> __DDR_CAR) & rid_mask; + int bar = (i >> __DDR_BAR) & rid_mask; + int rar = (i >> __DDR_RAR) & rid_mask; + int icar = (i >> __DDR_ICAR) & 0x1; + int ibar = (i >> __DDR_IBAR) & 0x1; + int irar = (i >> __DDR_IRAR) & 0x1; + int pall = (i >> __DDR_PALL) & 0x1; + int ap = (i >> __DDR_AP) & 0x1; + int bc = (i >> __DDR_BL4) & 0x1; + + switch(ddr_code) + { + case __WRITE: + printf("WR r%d%s r%d%s%s%s", bar, ibar?"++":"", car, icar?"++":"", + ap?" AP":"",bc?" BC":""); + break; + case __READ: + printf("RD r%d%s r%d%s%s%s", bar, ibar?"++":"", car, icar?"++":"", + ap?" AP":"",bc?" BC":""); + break; + case __PRE: + printf("PRE r%d%s%s", bar, ibar?"++":"", pall?" PALL":""); + break; + case __ACT: + printf("ACT r%d%s r%d%s", bar, ibar?"++":"", rar, irar?"++":""); + break; + case __ZQ: + printf("ZQ"); + break; + case __REF: + printf("REF"); + break; + case __NOP: + printf("NOP"); + break; + } +} + +void print_bits(size_t const size, void const * const ptr) +{ + unsigned char *b = (unsigned char*) ptr; + unsigned char byte; + int i, j; + + for (i=size-1;i>=0;i--) + { + for (j=7;j>=0;j--) + { + byte = (b[i] >> j) & 1; + printf("%u", byte); + } + } + printf("\n"); +} diff --git a/sources/api/instruction.h b/sources/api/instruction.h new file mode 100644 index 0000000..1be8691 --- /dev/null +++ b/sources/api/instruction.h @@ -0,0 +1,200 @@ +#ifndef INSTRUCTION_H +#define INSTRUCTION_H + +#include <stdint.h> +#include <stdlib.h> +// SoftMC decode +#define __OP_CODE 59 +#define __FU_CODE 48 +#define __IS_BR 62 +#define __IS_DDR 63 +#define __IS_MISC 61 +#define __IS_MEM 60 +#define __IS_BW 59 +// EXE decode +#define __RS1 0 +#define __RS2 4 +#define __RT 20 +#define __IMD1 4 +#define __IMD2 0 +#define __IMD3 24 +#define __BR_TGT 8 +#define __J_TGT 0 +#define __SLP_AMT 0 +// DDR decode +#define __DDR_CMD 12 +#define __DDR_CAR 4 +#define __DDR_BAR 0 +#define __DDR_RAR 4 +#define __DDR_IBAR 10 +#define __DDR_ICAR 11 +#define __DDR_IRAR 11 +#define __DDR_PALL 11 +#define __DDR_AP 9 +#define __DDR_BL4 8 +// EXE function codes +#define __ADD 0 +#define __ADDI 1 +#define __SUB 2 +#define __SUBI 3 +#define __MV 4 +#define __SRC 5 +#define __LI 6 +#define __LDWD 7 +#define __LDPC 8 +#define __SRE 0x100 +#define __SRX 0x101 +#define __BL 0 +#define __BEQ 1 +#define __JUMP 2 +#define __SLEEP 3 +#define __INFO 0 // Various information about upcoming block +#define __AND 0 +#define __OR 1 +#define __XOR 2 +#define __LD 0 +#define __ST 1 +// DDR function codes +#define __WRITE 8 +#define __READ 9 +#define __PRE 10 +#define __ACT 11 +#define __ZQ 12 +#define __REF 13 +#define __NOP 15 + +// 64 bit instructions +typedef uint64_t Inst; +// 16 bit mini ddr-instructions +typedef uint16_t Mininst; + +//Counter types for LDPC Inst +enum class PC_TYPE { WRITE, READ, PRE, ACT, ZQ, REF, CYC }; + +/** + * Load a word from memory into rt + * rt = [rb] + offset + * To handle structural hazards, the API adds a + * "buffer" NOP instruction following loads. + * @param rb register holding the base address + * @param offset memory address offset + * @param rt register to load the value with + */ +Inst SMC_LD(int rb, int offset, int rt); +/** + * Store a word to memory + * [rb] + offset = rt + * @param rb register holding the base address + * @param offset memory address offset + * @param rv register holding the value to store + */ +Inst SMC_ST(int rb, int offset, int rv); + +Inst SMC_AND(int rs1, int rs2, int rt); +Inst SMC_OR(int rs1, int rs2, int rt); +Inst SMC_XOR(int rs1, int rs2, int rt); + +Inst SMC_ADD(int rs1, int rs2, int rt); +Inst SMC_ADDI(int rs1, uint32_t imd, int rt); +Inst SMC_SUB(int rs1, int rs2, int rt); +Inst SMC_SUBI(int rs1, uint32_t imd, int rt); +Inst SMC_LI(uint32_t imd, int rt); +Inst SMC_MV(int rs1, int rt); +/** + * Shift right circular, shift one bit to right and copy + * the rightmost bit to the leftmost bit of the result. + * @param rs1 register to shift + * @param rt register to load the shifted value into + */ +Inst SMC_SRC(int rs1, int rt); +/** + * Move 32 bit data to wide register's specified offset + * @param rs1 source register where 32-bit data resides + * @param off 32-bit offset (e.g. 0 = bytes(0,4) - 5 = bytes(20,24)) + */ +Inst SMC_LDWD(int rs1, int off); +Inst SMC_LDPC(PC_TYPE pc_type, int rt); +Inst SMC_BL(int rs1, int rs2, int tgt); +Inst SMC_BEQ(int rs1, int rs2, int tgt); +Inst SMC_JUMP(int tgt); +Inst SMC_END(); +Inst SMC_INFO(int rdcnt); + +/** + * Wait for a specified amount of fabric cycles (6ns by default) before + * executing the next instruction + * @param samt how many cycles to wait for, must be greater than 2 + */ +Inst SMC_SLEEP(uint32_t samt); + +/** + * Generate a DDR-WR command + * @param bar bank address register ID + * @param ibar increment BAR (BAR=BAR+BASR) after issuing the write + * @param car column address register ID + * @param icar increment CAR (CAR=CAR+CASR) after issuing the write + * @param BL4 burst-chop write + * @param ap auto-precharge after issuing write + */ +Mininst SMC_WRITE(int bar, int ibar, int car, int icar, int BL4, int ap); +/** + * Generate a DDR-RD command + * @param bar bank address register ID + * @param ibar increment BAR (BAR=BAR+BASR) after issuing the read + * @param car column address register ID + * @param icar increment CAR (CAR=CAR+CASR) after issuing the read + * @param BL4 burst-chop read + * @param ap auto-precharge after issuing read + */ +Mininst SMC_READ(int bar, int ibar, int car, int icar, int BL4, int ap); +/** + * Generate a DDR-PRE command + * @param bar bank address register ID + * @param ibar increment BAR (BAR=BAR+BASR) after issuing the write + * @param pall precharge all banks + */ +Mininst SMC_PRE(int bar, int ibar, int pall); +/** + * Generate a DDR-RD command + * @param bar bank address register ID + * @param ibar increment BAR (BAR=BAR+BASR) after issuing the activate + * @param rar row address register ID + * @param irar increment RAR (RAR=RAR+RASR) after issuing the activate + */ +Mininst SMC_ACT(int bar, int ibar, int rar, int irar); +/** + * Generate a zq-calibration command + */ +Mininst SMC_ZQ(); +/** + * Generate a refresh command + */ +Mininst SMC_REF(); +/** + * Generate a no-operation command + */ +Mininst SMC_NOP(); +/** + * Enters Self-Refresh Mode + */ +Inst SMC_SRE(); +/** + * Exits Self-Refresh Mode + */ +Inst SMC_SRX(); +/** + * Packs four DDR commands together + */ +Inst __pack_mininsts(Mininst i1, Mininst i2, Mininst i3, Mininst i4); + +int is_branch(Inst i); +int is_conditional(Inst i); +int is_load(Inst i); +int is_ddr(Inst i); +int is_sleep(Inst i); +int is_ddr_read(Inst i); +void decode_inst(Inst inst); +void decode_ddr(Mininst i); +void print_bits(size_t const size, void const * const ptr); + +#endif diff --git a/sources/api/lexyacc/Makefile b/sources/api/lexyacc/Makefile new file mode 100644 index 0000000..67a05fa --- /dev/null +++ b/sources/api/lexyacc/Makefile @@ -0,0 +1,29 @@ +program_NAME := smc_parser +program_CXX_SRCS := lex.yy.c y.tab.c $(wildcard ../*.c*) +program_CXX_OBJS := ${program_CXX_SRCS:.cpp=.o} +program_OBJS := $(program_CXX_OBJS) +program_INCLUDE_DIRS := ../ ../../../boost-lib +program_LIBRARY_DIRS := +program_LIBRARIES := +CPPFLAGS += -g -std=c++11 -pthread + +CPPFLAGS += $(foreach includedir,$(program_INCLUDE_DIRS),-I$(includedir)) +LDFLAGS += $(foreach librarydir,$(program_LIBRARY_DIRS),-L$(librarydir)) +LDFLAGS += $(foreach library,$(program_LIBRARIES),-l$(library)) + +CC=g++ + +.PHONY: all clean distclean + +all: $(program_NAME) + +$(program_NAME): $(program_OBJS) lex.yy.c + $(CC) $(CPPFLAGS) $(program_OBJS) -o $(program_NAME) $(LDFLAGS) + +lex.yy.c: y.tab.c smc.l + flex smc.l + +y.tab.c: smc.y + yacc -d smc.y + +distclean: clean diff --git a/sources/api/lexyacc/example.smc b/sources/api/lexyacc/example.smc new file mode 100644 index 0000000..f9cc4f6 --- /dev/null +++ b/sources/api/lexyacc/example.smc @@ -0,0 +1,102 @@ +// Initialize register ids +// Similar to #define +casr = 0 +basr = 1 +rasr = 2 +bar = 2 +rar = 3 +car = 4 +pattern_reg = 5 +no_row_reg = 6 +no_bank_reg = 7 + +// Begin writing the program +LI no_row_reg 32768 // Write 2^15 to no_row_Reg +LI no_bank_reg 16 // write 16 to no_bank_reg + +// Set stride registers (not required if you +// are not going to increment address registers +// with ddr commands) +LI casr 8 // set column address stride to 8 +LI basr 1 // set bas to 1 +LI rasr 1 // set ras to 1 + +LI bar 0 // Initialize bank address register + +// Init pattern register, we are going to fill +// dram with 0xffs. +LI pattern_reg 0xffffffff + +// Load the data register with the pattern +// second argument here is the word offset +// we are writing to in the data register +// e.g. LDWD 0 pattern reg fills the first four bytes + +LDWD 0 pattern_reg +LDWD 1 pattern_reg +LDWD 2 pattern_reg +LDWD 3 pattern_reg +LDWD 4 pattern_reg +LDWD 5 pattern_reg +LDWD 6 pattern_reg +LDWD 7 pattern_reg +LDWD 8 pattern_reg +LDWD 9 pattern_reg +LDWD 10 pattern_reg +LDWD 11 pattern_reg +LDWD 12 pattern_reg +LDWD 13 pattern_reg +LDWD 14 pattern_reg +LDWD 15 pattern_reg + +BANK_BEGIN: // Add label to loop over banks +LI rar 0 // Reload rar per bank loop + +ROW_BEGIN: // Add label to loop over rows +LI car 0 // Reload car per row loop + +PRE bar // PREcharge bar, don't increment, no precharge-all +WAIT 7 // Insert seven cycles of NOPs + +ACT bar rar // ACTivate bar, rar +WAIT 7 + +// This is a basic primitive that can be used +// to replicate instructions-commands +// this block replicates the WR command 128 times +// and adds it to the program +times 128 { + WR bar car++ + WAIT 3 +} + +// Reload car, we are now going to read from DRAM +LI car 0 + +// Wait tWR, might not be precise. +WAIT 8 +PRE bar +WAIT 7 + +ACT bar rar +WAIT 7 + +times 128 { + RD bar car++ + WAIT 3 +} + +WAIT 8 +PRE bar + +// Increment rar per row loop +ADDI rar rar 1 +// if rar < no_row_reg, execute the "row loop" once more +BL ROW_BEGIN rar no_row_reg + +ADDI bar bar 1 +BL BANK_BEGIN bar no_bank_reg + +// Tell softmc that the program has ended +// This is important! +END
\ No newline at end of file diff --git a/sources/api/lexyacc/smc.l b/sources/api/lexyacc/smc.l new file mode 100644 index 0000000..4d9e478 --- /dev/null +++ b/sources/api/lexyacc/smc.l @@ -0,0 +1,77 @@ +%{ + #include <stdlib.h> + #include "instruction.h" + #include "y.tab.h" +%} + +%% + +"//"[^\n]* ; /* Skip comments */ +[%\t ]+ ; /* Skip whitespace */ + +0x[0-9a-f]+ { + yylval.lit = yytext+2; + return HEX_NUMBER; + } + +[0-9]+ { + yylval.num = atoi(yytext); + return DEC_NUMBER; + } + +r[0-9][0-9]? { + yylval.num = atoi(++yytext); /* Skip the first character? */ + return REG_ID; + } + +"wait"|"WAIT" return WAIT; + +"add"|"ADD" return ADD; +"addi"|"ADDI" return ADDI; +"sub"|"SUB" return SUB; +"subi"|"SUBI" return SUBI; +"mov"|"MOV" return MOV; +"src"|"SRC" return SRC; +"li"|"LI" return LI; +"ldwd"|"LDWD" return LDWD; + +"bl"|"BL" return BRL; +"beq"|"beq" return BREQ; +"jmp"|"JMP" return JMP; + +"wr"|"WR" return WR; +"rd"|"RD" return RD; +"act"|"ACT" return ACT; +"pre"|"PRE" return PRE; +"ref"|"REF" return REF; +"zq"|"ZQ" return ZQ; +"end"|"END" return END; + + +"++" return INCREMENT; +"AP"|"ap" return AUTO_PRECHARGE; +"BC"|"bc" return BURST_CHOP; +"PA"|"pa" return PRECHARGE_ALL; + +"times" return GENFOR; +"ENDPROGRAM" return ENDPROGRAM; + + [a-z_]+ { + char* var_name = (char*) malloc((yyleng+1)*sizeof(char)); + strcpy(var_name, yytext); + yylval.lit = var_name; + return VARIABLE; + } + + [A-Z_]+ { + char* lbl_name = (char*) malloc((yyleng+1)*sizeof(char)); + strcpy(lbl_name, yytext); + yylval.lit = lbl_name; + return LABEL; + } + + . return *yytext; + +%% + +int yywrap(void) { return 1; } diff --git a/sources/api/lexyacc/smc.y b/sources/api/lexyacc/smc.y new file mode 100644 index 0000000..f9ff06d --- /dev/null +++ b/sources/api/lexyacc/smc.y @@ -0,0 +1,427 @@ +%{ + #include "prog.h" + #include "instruction.h" + #include <iostream> + #include <stdlib.h> + #include <stdint.h> + #include <string> + #include <string.h> + #include <queue> + #include <map> + #include <stack> + + extern char* yytext; + extern int yyleng; + extern int yylex(void); + extern "C" void yyerror(char const*); + + int block_levels = 0; + + Program* prog; + std::stack <Program*> prog_addr_stack; + void process_ddr_command_queue(); + std::stack <uint32_t> num_generate_stack; + std::queue <Mininst> ddr_command_queue; + std::map<std::string, uint32_t> variable_map; +%} + +%start program + +%union { void* prg; + Mininst mini_i_type; + Inst i_type; + uint32_t num; + char* lit; + } + +%token <lit> HEX_NUMBER +%token <num> DEC_NUMBER +%token <num> REG_ID +%token <lit> LABEL +%token <lit> VARIABLE + +/* Instruction OP-Codes */ +%token WAIT + +%token ADD +%token ADDI +%token SUB +%token SUBI +%token MOV +%token SRC +%token LI +%token LDWD + +%token BRL +%token BREQ +%token JMP + +%token WR +%token RD +%token ACT +%token PRE +%token REF +%token ZQ +%token END + +/* Misc. operators */ +%token INCREMENT +%token AUTO_PRECHARGE +%token BURST_CHOP +%token PRECHARGE_ALL + +/* Other Expressions */ +%token GENFOR +%token ENDPROGRAM + +%type <num> gen_begin +%type <num> immediate +%type <i_type> instruction +%type <mini_i_type> ddr_command +%type <num> address_register +%type <num> reg_id +%type <num> rw_args +%type <num> pre_args + +%% + +program: + expression_list + | program ENDPROGRAM + { + } + ; + +expression_list: + expression {} + | expression_list expression {} + +expression: + instruction + { + // See if DDR command queue is empty + if (ddr_command_queue.size() > 0) + { + printf("WARNING: I just appended some NOPs to your code\n"); + int nops_to_append = 4 - ddr_command_queue.size(); + Mininst ddr_cmd_batch[4]; + for (int i = 0 ; i < ddr_command_queue.size() ; i++) + { + ddr_cmd_batch[i] = ddr_command_queue.front(); + ddr_command_queue.pop(); + } + for (int i = ddr_command_queue.size() ; i < 4 ; i++) + ddr_cmd_batch[i] = SMC_NOP(); + prog->add_inst(__pack_mininsts(ddr_cmd_batch[0],ddr_cmd_batch[1], + ddr_cmd_batch[2],ddr_cmd_batch[3])); + } + prog->add_inst($1); + } + | branch + { + + } + | pseudo_instruction + { + + } + | LABEL ':' + { + prog->add_label(std::string($1)); + } + | ddr_command + { + ddr_command_queue.push($1); + process_ddr_command_queue(); + } + | definition + { + + } + | gen_begin + { + num_generate_stack.push($1); + } + | '}' // end of a generate block + { + if(block_levels == 0) // pop an error: no generate blocks present, cannot end a block definition + printf("Expected \'}\' somewhere.\n"); // TODO change this with a correct error displaying function + else + { + Program* previous_in_stack = prog_addr_stack.top(); + prog_addr_stack.pop(); + int num_gen = num_generate_stack.top(); + num_generate_stack.pop(); + for (int i = 0 ; i < num_gen ; i++) + previous_in_stack->add_below(*prog); + free(prog); + prog = previous_in_stack; + } + } + ; + +instruction: + ADD reg_id reg_id reg_id + { + $$ = SMC_ADD($3, $4, $2); + } + | ADDI reg_id reg_id immediate + { + $$ = SMC_ADDI($3, $4, $2); + } + | SUB reg_id reg_id reg_id + { + $$ = SMC_SUB($3, $4, $2); + } + | SUBI reg_id reg_id immediate + { + $$ = SMC_SUBI($3, $4, $2); + } + | MOV reg_id reg_id + { + $$ = SMC_MV($3, $2); + } + | SRC reg_id reg_id + { + $$ = SMC_SRC($3, $2); + } + | LI reg_id immediate + { + $$ = SMC_LI($3, $2); + } + | LDWD immediate reg_id + { + $$ = SMC_LDWD($3, $2); + } + | END + { + $$ = SMC_END(); + } + ; + +branch: + BRL LABEL reg_id reg_id + { + prog->add_branch(prog->BR_TYPE::BL, $3, $4, $2); + } + | BREQ LABEL reg_id reg_id + { + prog->add_branch(prog->BR_TYPE::BEQ, $3, $4, $2); + } + | JMP LABEL + { + prog->add_branch(prog->BR_TYPE::JUMP, 0, 0, $2); + } + ; + +pseudo_instruction: + WAIT immediate + { + // currently functions only by adding NOPs + // TODO depending on the # of cycles specified + // we can instead generate a "loop" which enables + // us to wait for more with less instructions. + for(int i = 0 ; i < $2 ; i++) + ddr_command_queue.push(SMC_NOP()); + process_ddr_command_queue(); + } + ; + +immediate: + HEX_NUMBER + { + $$ = (uint32_t) strtol($1, NULL, 16); + } + | DEC_NUMBER + { + $$ = $1; + } + ; +ddr_command: + WR address_register address_register rw_args + { + int reg1_id = $2; + int reg2_id = $3; + int do_increment1 = reg1_id >= 100 ? 1 : 0; + int do_increment2 = reg2_id >= 100 ? 1 : 0; + if(do_increment1) + reg1_id -= 100; + if(do_increment2) + reg2_id -= 100; + int do_ap = $4 == 3 ? 1 : $4 == 1 ? 1 : 0; + int do_bc = $4 == 3 ? 1 : $4 == 2 ? 1 : 0; + $$ = SMC_WRITE(reg1_id, do_increment1, reg2_id, do_increment2, do_bc, do_ap); + } + | RD address_register address_register rw_args + { + int reg1_id = $2; + int reg2_id = $3; + int do_increment1 = reg1_id >= 100 ? 1 : 0; + int do_increment2 = reg2_id >= 100 ? 1 : 0; + if(do_increment1) + reg1_id -= 100; + if(do_increment2) + reg2_id -= 100; + int do_ap = $4 == 3 ? 1 : $4 == 1 ? 1 : 0; + int do_bc = $4 == 3 ? 1 : $4 == 2 ? 1 : 0; + $$ = SMC_READ(reg1_id, do_increment1, reg2_id, do_increment2, do_bc, do_ap); + } + | ACT address_register address_register + { + int reg1_id = $2; + int reg2_id = $3; + int do_increment1 = reg1_id >= 100 ? 1 : 0; + int do_increment2 = reg2_id >= 100 ? 1 : 0; + if(do_increment1) + reg1_id -= 100; + if(do_increment2) + reg2_id -= 100; + $$ = SMC_ACT(reg1_id, do_increment1, reg2_id, do_increment2); + } + | PRE address_register pre_args + { + int reg_id = $2; + int do_increment = reg_id>=100 ? 1 : 0; + if(do_increment) + reg_id -= 100; + $$ = SMC_PRE(reg_id, do_increment, $3); + } + | REF + { + $$ = SMC_REF(); + } + | ZQ + { + $$ = SMC_ZQ(); + } + ; + +address_register: + reg_id + { + $$ = $1; + } + | reg_id INCREMENT + { + $$ = 100 + $1; + } + ; + +definition: + VARIABLE '=' immediate + { + std::string key = std::string($1); + variable_map[key] = $3; + } + ; + +reg_id: + REG_ID + { + $$ = $1; + } + | VARIABLE + { + std::string key = std::string($1); + if (variable_map.find(key) == variable_map.end()) + { + // TODO proper error handling + std::cout << "Variable " << key << " is not defined." << std::endl; + $$ = 0; + } else + $$ = variable_map[key]; + } + ; + +rw_args: + %empty + { + $$ = 0; + } + | AUTO_PRECHARGE + { + $$ = 1; + } + | BURST_CHOP + { + $$ = 2; + } + | AUTO_PRECHARGE BURST_CHOP + { + $$ = 3; + } + | BURST_CHOP AUTO_PRECHARGE + { + $$ = 3; + } + ; + +pre_args: + %empty + { + $$ = 0; + } + | PRECHARGE_ALL + { + $$ = 1; + } + ; + +gen_begin: + GENFOR immediate '{' + { + // See if DDR command queue is empty + if (ddr_command_queue.size() > 0) + { + printf("WARNING: I just appended some NOPs to your code\n"); + int nops_to_append = 4 - ddr_command_queue.size(); + Mininst ddr_cmd_batch[4]; + for (int i = 0 ; i < ddr_command_queue.size() ; i++) + { + ddr_cmd_batch[i] = ddr_command_queue.front(); + ddr_command_queue.pop(); + } + for (int i = ddr_command_queue.size() ; i < 4 ; i++) + ddr_cmd_batch[i] = SMC_NOP(); + prog->add_inst(__pack_mininsts(ddr_cmd_batch[0],ddr_cmd_batch[1], + ddr_cmd_batch[2],ddr_cmd_batch[3])); + } + prog_addr_stack.push(prog); + prog = new Program(); + block_levels++; + $$ = $2; // Number of times this block is generated + } + ; + +%% + +void process_ddr_command_queue() +{ + while (ddr_command_queue.size() >= 4) + { + Mininst ddr_cmd_batch[4]; + for (int i = 0 ; i < 4 ; i++) + { + ddr_cmd_batch[i] = ddr_command_queue.front(); + ddr_command_queue.pop(); + } + prog->add_inst(__pack_mininsts(ddr_cmd_batch[0],ddr_cmd_batch[1], + ddr_cmd_batch[2],ddr_cmd_batch[3])); + } +} + +void yyerror (char const *s) { + fprintf (stderr, "%s\n", s); +} + +int main(int argc, char* argv[]) +{ + if(argc != 2){ + printf("I need you to supply me with the filename!\n"); + exit(1); + } + prog = new Program(); + yyparse(); + prog->save_bin(std::string(argv[1])); +} + diff --git a/sources/api/platform.cpp b/sources/api/platform.cpp new file mode 100644 index 0000000..2dae891 --- /dev/null +++ b/sources/api/platform.cpp @@ -0,0 +1,296 @@ +#include <stdlib.h> +#include <string.h> +#include <sys/time.h> +#include <stdlib.h> +#include <cassert> +#include <stdint.h> +#include <stdio.h> +#include <thread> +#include <iostream> +#include <unistd.h> +#include <boost/lockfree/spsc_queue.hpp> + +#include "platform.h" +#include "board.h" +#include "prog.h" + +[[maybe_unused]] static int consume_total = 0; +[[maybe_unused]] static int receive_total = 0; + +SoftMCPlatform::SoftMCPlatform() +{ + // 32 KB large read buffer + is_dummy = false; + xdma_recv_buf = malloc(32*1024); + + #ifdef PYSMC + py_data_buffer = (uint8_t*)malloc(32*1024*sizeof(uint8_t)); + #endif +} + +SoftMCPlatform::SoftMCPlatform(bool sandbox) +{ + // 32 KB large read buffer + is_dummy = sandbox; + xdma_recv_buf = malloc(32*1024); +} + +SoftMCPlatform::~SoftMCPlatform(){ + if (receiver.joinable()) + receiver.join(); + + if (xdma_recv_buf) + free(xdma_recv_buf); + + if (instr_buf) + free(instr_buf); + + if (iface) + delete iface; + + #ifdef PYSMC + if (py_data_buffer) + free(py_data_buffer); + #endif +} + +int SoftMCPlatform::init(){ + if(is_dummy) + { + instr_buf = malloc(INSTR_BUF_SIZE); + memset(instr_buf, 0, INSTR_BUF_SIZE); + return SOFTMC_SUCCESS; + } + else + { + instr_buf = malloc(INSTR_BUF_SIZE); + memset(instr_buf, 0, INSTR_BUF_SIZE); + + iface = new BoardInterface(BoardInterface::IFACE::XDMA); + if(!iface -> init()) + return SOFTMC_SUCCESS; + else + return SOFTMC_ERR; + } +} + +/** + * This sends a 256 bit data which has it's + * 33rd bit set to '1'. + */ +void SoftMCPlatform::reset_fpga() +{ + if(is_dummy) + { + return; + } + else + { + ((uint8_t*) instr_buf)[8] = (uint8_t) 1; + int sent = iface -> sendData(instr_buf, 32 /*in bytes*/); + // We do not need to zero out the whole buffer + memset(instr_buf, 0, 32); + if(sent) + std::cerr << "Could not reset the FPGA!" << std::endl; + else + std::cout << "Successfully reset the FPGA!" << std::endl; + } +} + +void SoftMCPlatform::execute(Program &prog) +{ + if(is_dummy) + { + [[maybe_unused]] uint64_t* iseq = (uint64_t*) prog.get_inst_array(); + int bytes = prog.size(); + assert (bytes <= INSTR_BUF_SIZE/4 && " too many instructions in the buffer, the limit is 2048."); + return; + } + else + { + uint64_t* iseq = (uint64_t*) prog.get_inst_array(); + uint64_t* temp_ptr = (uint64_t*) instr_buf; + int bytes = prog.size(); + assert (bytes <= INSTR_BUF_SIZE/4 && " too many instructions in the buffer, the limit is 2048."); + + for(int i = 0 ; i < bytes/8 ; i++) + temp_ptr[i*4] = iseq[i]; + + if(receiver.joinable()) + receiver.join(); + + receiver = std::thread(&SoftMCPlatform::consumeData, this); + int sent = iface -> sendData(instr_buf, bytes*4 /*in bytes*/); + memset(instr_buf, 0, bytes*4); + free(iseq); + assert(!sent && "could not send instructions"); + } +} + +void SoftMCPlatform::consumeData() +{ + while(true) + { + int xdma_read_intent = 32*1024; + int xdma_read_size = xdma_read_intent; + int recvd = iface -> recvData((void*)xdma_recv_buf, xdma_read_size); + if(recvd == 0) + break; + // xdma read size in words + // If we did not read 32KBs then the program probably ended + // and the last transfer is trash so we discard it + if(recvd != xdma_read_intent) + xdma_read_size = recvd-32; + xdma_read_size /= 4; + int total_size = xdma_read_size; + int pushsz = api_recv_buf.push(((int*) xdma_recv_buf), xdma_read_size); + while(pushsz < total_size) + pushsz += api_recv_buf.push(((int*) xdma_recv_buf) + pushsz, xdma_read_size = (total_size - pushsz)); + +// printf("SoftMCPlatform::consumedata(): spsc filled with %d beats of data\n", pushsz); +// printf("Consume total: %d\n",++consume_total); + + assert(pushsz == total_size && + "Unexpected amount of data pushed into spsc\n"); + + if(recvd != xdma_read_intent) + break; + + } +} + +/** +* Try to read param(size) bytes from FPGA, function will block until +* all data is read +* @param recv_buf where to copy read data +* @param size number of bytes to read +* returns the number of bytes read on success +*/ +int SoftMCPlatform::receiveData(void* recv_buf, int size){ + if(is_dummy) + { + assert(size>0 && size%4 == 0 && "size is expected to be a multiple of four\n"); + size /= 4; + int * my_buf = (int *) recv_buf; + for(int i = 0; i < size ; ++i) { + my_buf[i] = 0x0; + } + return size * 4; + } + else + { + assert(size>0 && size%4 == 0 && "size is expected to be a multiple of four\n"); + + size /= 4; + int total_size = size; + int rdsz = api_recv_buf.pop((int*) recv_buf, size); + while (rdsz < total_size) + rdsz += api_recv_buf.pop(((int*) recv_buf) + rdsz, size = (total_size-rdsz)); + + assert(rdsz == total_size && "Unexpected amount of data popped from spsc\n"); + return total_size*4; + } +} + +#ifdef PYSMC +int SoftMCPlatform::py_receiveData(int size){ + if (size > 32 * 1024) + { + std::cerr << "Python version only supports read buffer size of up to 32KB!" << std::endl; + return 0; + } + + if(is_dummy) + { + assert(size>0 && size%4 == 0 && "size is expected to be a multiple of four\n"); + size /= 4; + int * my_buf = (int *) py_data_buffer; + for(int i = 0; i < size ; ++i) { + my_buf[i] = 0x0; + } + return size * 4; + } + else + { + assert(size>0 && size%4 == 0 && "size is expected to be a multiple of four\n"); + + size /= 4; + int total_size = size; + int rdsz = api_recv_buf.pop((int*) py_data_buffer, size); + while (rdsz < total_size) + rdsz += api_recv_buf.pop(((int*) py_data_buffer) + rdsz, size = (total_size-rdsz)); + + assert(rdsz == total_size && "Unexpected amount of data popped from spsc\n"); + return total_size*4; + } +} + +py::memoryview SoftMCPlatform::get_buffer_memoryview() +{ + return py::memoryview::from_memory((uint64_t*) py_data_buffer, sizeof(uint8_t) * 32 * 1024, true); +} +#endif + +int SoftMCPlatform::count_bitflips_in_row(unsigned char comp_pattern){ + int num_bitflips = 0; + unsigned char buf[8192]; + receiveData(buf, 8192); // read one row each iteration + + for(int j = 0 ; j < 8192 ; j++){ + if(comp_pattern != buf[j]) + { + for(int i = 0 ; i < 8 ; i++) + { + if(((comp_pattern >> i) & 1) != ((buf[j] >> i) & 1)) + num_bitflips ++; + } + } + } + + return num_bitflips; +} + +void SoftMCPlatform::set_aref(const bool on) +{ + if(is_dummy) + { + std::cout << (on ? "Enabled" : "Disabled") << " autorefresh!" << std::endl; + return; + } + else + { + ((uint8_t*) instr_buf)[8] = (uint8_t) 0x8; + ((uint8_t*) instr_buf)[0] = on; + int sent = iface -> sendData(instr_buf, 32 /*in bytes*/); + // We do not need to zero out the whole buffer + memset(instr_buf, 0, 32); + + if(sent) + std::cerr << "Could not set auto refresh!" << std::endl; + // else + // std::cout << (on ? "Enabled" : "Disabled") << " autorefresh!" << std::endl; + } +} + +void SoftMCPlatform::readRegisterDump() +{ + if(is_dummy) + return; + + /** The author decided to use printfs explicitly within this function + * because printing unsigned hex bytes is uglier with stdio + */ + uint8_t readData[64]; + this -> receiveData((void*)readData, 64); // first read wdata content + printf("WDATA: 0x"); + for(int i = 63 ; i >= 0 ; i--) + printf("%x", readData[i]); + printf("\n"); + this -> receiveData((void*)readData, 64); // read register content + for(int r = 0 ; r < 16 ; r++) + { + printf("R%d: 0x", r); + printf("%x", ((uint32_t*)readData)[r]); + printf("\n"); + } +}
\ No newline at end of file diff --git a/sources/api/platform.h b/sources/api/platform.h new file mode 100644 index 0000000..277dcdf --- /dev/null +++ b/sources/api/platform.h @@ -0,0 +1,83 @@ +#include "board.h" +#include "prog.h" +#include <thread> +#include <boost/lockfree/spsc_queue.hpp> + +#ifdef PYSMC +#include "ext/pybind11/include/pybind11/pybind11.h" +namespace py = pybind11; +#endif + +//ERROR CODES +#define SOFTMC_SUCCESS 0 +#define SOFTMC_ERR -1 +#define SOFTMC_NO_PLATFORM -2 +#define SOFTMC_ERR_OPEN_FPGA -3 +#define SOFTMC_NO_SUCH_FPGA -4 + +class SoftMCPlatform{ + #define INSTR_BUF_SIZE 32*2048 + #define API_BUF_SIZE 1024*1024*2 + public: + SoftMCPlatform(); + SoftMCPlatform(bool); + ~SoftMCPlatform(); + /** + * Initializes the whole platform + * @return SOFTMC_SUCCESS on sucessful initialization + */ + int init(); + /** + * Resets SoftMC logic, won't reset PCI-E endpoint or the PHY interface + */ + void reset_fpga(); + /** + * Sends SoftMC program to the FPGA board over PCI-E + * @param prog reference to the program object to send + */ + void execute(Program & prog); + /** + * Receive data from the FPGA board over PCI-E + * @param dst_buf pointer to the buffer that will receive the data + * @param num_words number of bytes to read + */ + int receiveData(void* dst_buf, int num_words); + + #ifdef PYSMC + int py_receiveData(int num_words); + #endif + + /** + * Compare data with a given data pattern (repeating bytes) + * and return number of bitflips in 8KB of data + * @param comp_pattern one byte data pattern to compare the read data + */ + int count_bitflips_in_row(unsigned char comp_pattern); + + /** + * Turn auto-refresh on-off + * @param on true to turn aref on false otherwise + */ + void set_aref(bool on); + + /** + * Used along with Program::dumpRegisters to read register content + */ + void readRegisterDump(); + + private: + bool is_dummy; + + BoardInterface *iface; + void* instr_buf; + std::thread receiver; + boost::lockfree::spsc_queue<int, boost::lockfree::capacity<API_BUF_SIZE/4>> api_recv_buf; + void* xdma_recv_buf; + void consumeData(); + + #ifdef PYSMC + public: + uint8_t* py_data_buffer = nullptr; + py::memoryview get_buffer_memoryview(); + #endif +}; diff --git a/sources/api/prog.cpp b/sources/api/prog.cpp new file mode 100644 index 0000000..781c079 --- /dev/null +++ b/sources/api/prog.cpp @@ -0,0 +1,675 @@ +#include "instruction.h" +#include "prog.h" +#include <string> +#include <vector> +#include <cassert> +#include <iostream> +#include <fstream> +#include <bitset> +#include <algorithm> +#include <sys/wait.h> +#include <sys/stat.h> +#include <unistd.h> +#include <fcntl.h> +#include <string.h> + +Program::Program() +{ + dumpRegsCalled = false; +} + +Program::Program(std::string fname) +{ + std::cout << "WARNING: make sure that the executable \"smc_parser\" exists in the working directory" << std::endl; + + std::string correctFileName = "./smc_parser " + fname + " < " + fname + " >/dev/null"; + if (system(correctFileName.c_str()) != 0) + { + std::cerr << "Error parsing SMC program, abort." << std::endl; + exit(-1); + } + + // Read instruction binary data from fname.inst + std::ifstream binFile (fname + ".bin", std::ios::in | std::ios::binary); + int no_insts = 0; + // Read no of insts + binFile.read((char*) &no_insts, 4); + Inst insts[no_insts]; + // Read all insts + binFile.read((char*) insts, sizeof(Inst)*no_insts); + for(int i = 0 ; i < no_insts ; i++) + { + program.push_back(insts[i]); + spc++; + } + binFile.close(); + // Read branch label positions from fname.meta + std::ifstream metaFile (fname + ".meta", std::ios::in); + std::string line; + bool parseLabels = true; + while(getline(metaFile,line)) + { + if(parseLabels) + { + if (line == "-") + { + parseLabels = false; + continue; + } + std::string label = line.substr(0,line.find(" ")); + std::string target = line.substr(line.find(" ")+1,line.length()); + (labels)[label] = stoi(target); + } + else + { + std::string target = line.substr(0,line.find(" ")); + std::string label = line.substr(line.find(" ")+1,line.length()); + (branches)[stoi(target)] = label; + } + } + metaFile.close(); +} + + +void Program::add_label(std::string name) +{ + if(labels.count(name)) + std::cerr << "Trying to add label " << name << " multiple times!" << std::endl; + (labels)[name] = spc; +} +void Program::add_wait(int wait_cycles) +{ + for(; wait_cycles > 0 ; wait_cycles--) + { + minprogram.push_back(SMC_NOP()); + } +} + +void Program::add_mininst(Mininst mi, int wait_after) +{ + minprogram.push_back(mi); + for ( ; wait_after>0 ; wait_after--) + minprogram.push_back(SMC_NOP()); + // if (verbose) { + // std::cout << "Added new mininst with wait." << std::endl; + // for(auto& mininst : minprogram){ + // std::cout << std::hex << mininst << " "; + // } + // std::cout << std::endl; + // } +} + +void Program::pack_minprogram() +{ + // if (verbose) std::cout << "Packing minprogram" << std::endl; + while(minprogram.size() >= 4) + { + this->add_inst(__pack_mininsts(minprogram[0], minprogram[1], minprogram[2], minprogram[3])); + minprogram.erase(minprogram.begin(), minprogram.begin()+4); + } + + switch (minprogram.size()) + { + case 0: + return; + case 1: + this->add_inst(__pack_mininsts(minprogram[0], SMC_NOP(), SMC_NOP(), SMC_NOP())); + break; + case 2: + this->add_inst(__pack_mininsts(minprogram[0], minprogram[1], SMC_NOP(), SMC_NOP())); + break; + case 3: + this->add_inst(__pack_mininsts(minprogram[0], minprogram[1], minprogram[2], SMC_NOP())); + break; + } + // std::cerr << "WARNING: Number of mininsts is not multiple of four. " + // << "Appending "<< 4-minprogram.size() << " extra NOPs for alignment." + // << std::endl; + minprogram.erase(minprogram.begin(), minprogram.end()); +} + +void Program::add_inst(Inst i) +{ + program.push_back(i); + if (is_load(i)) + this->add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()); + spc += 1; +} + +void Program::add_inst(Mininst m1, Mininst m2, Mininst m3, Mininst m4) +{ + Inst i = (uint64_t) m4 << 48 | + (uint64_t) m3 << 32 | + (uint64_t) m2 << 16 | + m1; + program.push_back(i); + spc += 1; +} + +void Program::add_branch(BR_TYPE bt, int rs1, int rs2, std::string tgt) +{ + Inst place_holder = 2; + switch(bt) + { + case BR_TYPE::BL: + place_holder = SMC_BL(rs1, rs2, 0); + break; + case BR_TYPE::BEQ: + place_holder = SMC_BEQ(rs1, rs2, 0); + break; + case BR_TYPE::JUMP: + place_holder = SMC_JUMP(0); + break; + } + (branches)[spc] = tgt; + add_inst(place_holder); +} + +void Program::add_below(const Program &p) +{ + // move instructions from p to the end of + // this program + program.insert(program.end(), p.program.begin(), p.program.end()); + + // update branch targets and branch instruction pcs + for ( const auto &branch : (p.branches) ) + ((this->branches))[branch.first + this -> spc] = branch.second; + + for ( const auto &label : (p.labels) ) + ((this->labels))[label.first] = label.second + this -> spc; + + // update pc + this -> spc += p.spc; +} + +Inst* Program::get_inst_array() +{ + linear_analysis(); + insert_generated(); +// printf("Inserted auto-generated info instructions\n"); + preprocess_branches(); +// printf("Assigned branch targets to labels\n"); + //pretty_print(); + int bytes = size(); + Inst* arr = (Inst*) malloc(bytes); + int ind = 0; + for (auto & element : (program)) + arr[ind++] = element; + return arr; +} + +Inst* Program::get_insts() +{ + int bytes = size(); + Inst* arr = (Inst*) malloc(bytes); + int ind = 0; + for (auto & element : (program)) + arr[ind++] = element; + return arr; +} + +int Program::size() +{ + return program.size() * 8; +} + +void Program::preprocess_branches() +{ + for ( const auto &branch : (branches) ) { + Inst br = (program)[branch.first]; + int lbl = (labels)[branch.second]; +// printf("branch addr: %d branch label: %s lbl_adr: %d is_cond: %d\n", branch.first, branch.second.c_str(), +// lbl,is_conditional(br)); + assert(is_branch(br) && + "preprocess_branches(): expected a branch but got something else"); + uint64_t target_b = ((uint64_t)lbl) << 8; + uint64_t target_j = (uint64_t) lbl; + br |= is_conditional(br) ? target_b : target_j; + (program)[branch.first] = br; + } +} + +/** + * Currently only goes through the whole program + * to see if any constraint is violated + */ +void Program::linear_analysis() +{ + bool inSequence = false; + uint seqPc = 0; + int readCtr = 0; + for (uint pc = 0 ; pc < program.size() ; pc++) + { + Inst cur_inst = (program)[pc]; + if(!inSequence) + { + if(is_ddr(cur_inst)) + { + //printf("Program::linear_analysis(): enter segment at pc: %d\n", pc); + inSequence = true; + seqPc = pc; + readCtr = is_ddr_read(cur_inst); + } + }else + { + // we see a non-ddr inst, + // can commit whatever info we want + // related to the previous segment + if(!is_ddr(cur_inst) && !is_sleep(cur_inst)) + { + //printf("Program::linear_analysis(): exit segment at pc: %d with %d reads \n", pc, readCtr); + inSequence = false; + // TODO either directly insert into the vector + // or keep these records elsewhere + if(readCtr > 1024) + { + // TODO Add code that will warn user violently here. + // maybe even an assertion? + std::cerr << "WARNING: SoftMC won't be able to run your program timing-critically!" << std::endl; + }else if(readCtr > 0) + { + // Insert info-encapsulating packet into program + Inst warn_pipeline = SMC_INFO(readCtr); + (warnings)[seqPc] = warn_pipeline; + } + }else + { + readCtr += is_ddr_read(cur_inst); + } + } + } +} + + +void Program::insert_generated() +{ + // WARNING this assumes that map keys are iterated + // in ascending order + + std::vector<uint> warn_keys; + std::vector<Inst> warn_values; + + for ( const auto &warn : (this->warnings)) + { + warn_keys.push_back(warn.first); + warn_values.push_back(warn.second); + } + + while (warn_keys.size()>0) + { + // new branch and label mappings + uint warn_key = warn_keys.front(); + Inst warn_value = warn_values.front(); + warn_keys.erase(warn_keys.begin()); + warn_values.erase(warn_values.begin()); + + std::map<std::string, uint> n_labels; + std::map<uint, std::string> n_branches; + program.insert(program.begin() + warn_key, warn_value); + for ( const auto &branch : (this->branches) ) + { + // branch target > inserted instruction pc + if((labels)[branch.second] > warn_key) + { + (n_labels)[branch.second] = (labels)[branch.second] + 1; + }else + { + (n_labels)[branch.second] = (labels)[branch.second]; + } + // branching instruction pc > inserted instruction pc + if(branch.first >= warn_key) + { + (n_branches)[branch.first + 1] = branch.second; + }else + { + (n_branches)[branch.first] = branch.second; + } + } + for (uint i = 0 ; i < warn_keys.size() ; i++) + warn_keys[i] += 1; + + labels = n_labels; + branches = n_branches; + } +} + +void Program::pretty_print() +{ + for (uint pc = 0 ; pc < program.size() ; pc++) + { + for ( const auto &label : (labels) ) + { + if(label.second == pc) + std::cout << label.first << std::endl; + } + printf("%04d: ", pc); + decode_inst((program)[pc]); + for ( const auto &branch : (branches) ) + { + if(branch.first == pc) + std::cout << " (TARGET LABEL: " << branch.second << ")"; + } + printf("\n"); + } + +} + +void Program::bin_dump() +{ + for (uint pc = 0 ; pc < program.size() ; pc++) + { + for ( const auto &label : (labels) ) + { + if(label.second == pc) + std::cout << label.first << std::endl; + } + std::cout << "PC: " << pc << " Inst: " << std::hex << (program)[pc] << std::dec << std::endl; + + for ( const auto &branch : (branches) ) + { + if(branch.first == pc) + std::cout << branch.second; + } + printf("\n"); + } +} + +void Program::save_bin(const std::string &fname) +{ + // Write instruction binary data to fname.inst + std::ofstream binFile (fname + ".bin", std::ios::out | std::ios::binary); + Inst* insts = this -> get_insts(); + int no_insts = spc; + binFile.write ((char*)&no_insts, sizeof(no_insts)); + binFile.write ((char*) insts, sizeof(Inst)*no_insts); + binFile.close(); + // Write branch label positions to fname.meta + std::ofstream metaFile (fname + ".meta", std::ios::out); + for ( const auto &label : (labels) ) + metaFile << label.first << " " << label.second << "\n"; + metaFile << "-" << "\n"; + for ( const auto &branch : (branches) ) + metaFile << branch.first << " " << branch.second << "\n"; + metaFile.close(); +} + +void Program::save_coe_here(const std::string &prj_dir) +{ + std::ofstream coeFile ("program.coe", std::ios::out | std::ios::binary); + uint64_t* insts = (uint64_t*) this -> get_inst_array(); + coeFile << "memory_initialization_radix=2;\n"; + coeFile << "memory_initialization_vector=\n"; + for(uint i = 0 ; i < program.size() ; i++){ + coeFile << std::bitset<64>(insts[i]); + if( i == (program.size()) -1){ + coeFile << ";"; + }else{ + coeFile << ",\n"; + } + } + coeFile.close(); +} + + +void Program::save_coe(const std::string &prj_dir) +{ + std::ofstream coeFile (prj_dir + "/VU095.sim/sim_1/behav/xsim/simmem.coe", std::ios::out | std::ios::binary); + std::ofstream coeFile2 (prj_dir + "/coe/simmem.coe", std::ios::out | std::ios::binary); + std::ofstream mifFile (prj_dir + "/VU095.sim/sim_1/behav/xsim/instr_blk_mem_sim.mif", std::ios::out | std::ios::binary); + uint64_t* insts = (uint64_t*) this -> get_inst_array(); + coeFile << "memory_initialization_radix=2;\n"; + coeFile2 << "memory_initialization_radix=2;\n"; + coeFile << "memory_initialization_vector=\n"; + coeFile2 << "memory_initialization_vector=\n"; + for(uint i = 0 ; i < program.size() ; i++){ + coeFile << std::bitset<64>(insts[i]); + coeFile2 << std::bitset<64>(insts[i]); + mifFile << std::bitset<64>(insts[i])<<"\n"; + if( i == (program.size()) -1){ + coeFile << ";"; + coeFile2 << ";"; + }else{ + coeFile << ",\n"; + coeFile2 << ",\n"; + } + } + coeFile.close(); + coeFile2.close(); + mifFile.close(); +} + + +void Program::dump_registers() +{ + if(dumpRegsCalled) + { + std::cerr << "You can use your trump card (dump_register) only once per program!" << std::endl; + exit(1); + } + dumpRegsCalled = true; + + this -> add_inst(SMC_LI(15,13)); // BAR + this -> add_inst(SMC_LI(0,14)); // RAR + this -> add_inst(SMC_LI(0,15)); // CAR + + this -> add_inst(__pack_mininsts(SMC_PRE(13,0,1), // PRECHARGE ALL + SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_ACT(13,0,14,0), // ACT B15 R0 + SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_WRITE(13,0,15,0,0,0), // WRITE WDATA CONTENT + SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_READ(13,0,15,0,0,0), // READ WDATA CONTENT + SMC_NOP(),SMC_NOP(),SMC_NOP())); + + for (int i = 0 ; i < 16 ; i++) // SWAP WDATA WITH REG VALUES + this -> add_inst(SMC_LDWD(i,i)); + + this -> add_inst(__pack_mininsts(SMC_WRITE(13,0,15,0,0,0), // WRITE WDATA CONTENT + SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_READ(13,0,15,0,0,0), // READ WDATA CONTENT + SMC_NOP(),SMC_NOP(),SMC_NOP())); + + this -> add_inst(__pack_mininsts(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP())); + this -> add_inst(__pack_mininsts(SMC_PRE(13,0,1), // PRECHARGE ALL + SMC_NOP(),SMC_NOP(),SMC_NOP())); +} + +bool Program::isDumpRegsCalled() +{ + return dumpRegsCalled; +} + +void Program::debug(const std::string &prj_dir, bool first) +{ + int pid = fork(); + if(pid == 0){ + + std::string source = prj_dir + "/source.tcl"; + std::string project = prj_dir + "/VU095.xpr"; + std::string arg; + int fd = open("/dev/null", O_WRONLY); + dup2(fd, 1); + if(first) + arg = "first"; + else + arg = "debug"; + + execl("/opt/Xilinx/Vivado/2018.2/bin/vivado", + "vivado","-mode","tcl","-nojournal","-applog","-log","/tmp/asmc.log", + "-source", source.c_str(), project.c_str(),"-tclargs", arg.c_str(), NULL); + + }else if(pid > 0){ + std::cout << "Debugger starting...\n"; + + mkfifo("/tmp/fifo_1", 0666); + int f1 = open("/tmp/fifo_1", O_WRONLY); + mkfifo("/tmp/fifo_2", 0666); + int f2 = open("/tmp/fifo_2", O_RDONLY); + + char buf[100]; + [[maybe_unused]] int rv; + + std::vector<std::string> vios; + std::string line, start, end, vio; + + start = "sim_tb_top.mem_model_x8.memModels_Ri1[0].memModel1[0].ddr4_model.always_diff_ck.if_diff_ck:VIOLATION:"; + end = "sim_tb_top.mem_model_x8.memModels_Ri1[0].memModel1[1].ddr4_model.always_diff_ck.if_diff_ck:VIOLATION:"; + + std::string msg, com, param,param2; + std::vector<std::string> commands {"reg", "mem", "time", "step", "run", + "until", "exit", "btwn","stat"}; + + int state = 0; + + while(state != 5){ + switch (state){ + case 0: + std::cout << "Enter command: "; + std::cin >> com; + if(std::find(commands.begin(), commands.end(), com) != commands.end()){ + if(com == "step" || com == "exit" || com == "time" || com == "stat"){ + msg = com + "_\n"; + }else if(com == "btwn"){ + std::cin >> param; + std::cin >> param2; + msg = com + "_" + param + "_" + param2 + "\n"; + }else{ + std::cin >> param; + msg = com + "_" + param + "\n"; + } + state = 1; + }else{ + std::cout << "Please enter a valid command...\n"; + } + break; + + case 1: + rv = write( f1, (char*)msg.c_str(), msg.size()); + memset(buf, 0, sizeof(buf)); + while(read(f2, buf, sizeof(buf)) == 0); + state = 2; + break; + + case 2: + if(com == "reg" || com == "mem") + std::cout << com << " " << param << ": " << buf; + else if(com == "btwn") + std::cout << "Time between "<< param << "-" << param2 << ": " << buf; + else if(com == "time") + std::cout << "Current time: " << buf; + else if(com == "stat") + std::cout << "Stats:\n" << buf; + else if(com == "exit"){ + state = 5; + break; + } + state = 3; + break; + + case 3: + std::ifstream logfile ("/tmp/asmc.log"); + while (std::getline(logfile, line)) + { + if(line.find(start) != std::string::npos) + { + vio = line; + if(std::find(vios.begin(), vios.end(), vio) != vios.end()) + continue; + std::cout << line.substr(91) << "\n"; + while (std::getline(logfile, line)) + { + if (line.find(end) != std::string::npos) + break; + std::cout << line << "\n"; + } + vios.insert(vios.begin(), vio); + } + } + logfile.close(); + state = 0; + break; + + } + } + close(f1); + close(f2); + wait(NULL); + remove("/tmp/fifo_1"); + remove("/tmp/fifo_2"); + remove("/tmp/asmc.log"); + std::cout << "Exiting debugger.\n"; + } +} + +void Program::debug(const std::string &prj_dir, const std::string &filename) +{ + int pid = fork(); + if(pid == 0){ + + std::string source = prj_dir + "/source.tcl"; + std::string project = prj_dir + "/VU095.xpr"; + int fd = open("/dev/null", O_WRONLY); + dup2(fd, 1); + execl("/opt/Xilinx/Vivado/2018.2/bin/vivado", + "vivado","-mode","tcl","-nojournal","-applog","-log","/tmp/asmc.log", + "-source", source.c_str(), project.c_str(),"-tclargs", "update", NULL); + + }else if(pid > 0){ + + std::string path, line, param, value, time_path, temp_path; + + path = prj_dir.substr(0, prj_dir.find("VU095")) + "phy_ddr4_ex/imports/"; + time_path = path + "timing_tasks.sv"; + temp_path = path + "temp.sv"; + + std::ifstream timing_tasks (time_path); + std::ifstream param_file (filename); + std::ofstream temp_file (temp_path); + + std::map <std::string, std::string> params; + + while (std::getline(param_file, line)){ + param = line.substr(0, line.find("=")); + param = param.substr(param.find_first_not_of(" \n\r\t\f\v"),param.find_last_not_of(" \n\r\t\f\v") + 1); + param = " SetTSArray (i" + param; + value = line.substr(line.find("=") + 1); + value = value.substr(value.find_first_not_of(" \n\r\t\f\v"),value.find_last_not_of(" \n\r\t\f\v") + 1); + value = std::string((7- value.size()), ' ') + value; + params[param] = value; + } + param_file.close(); + + while (std::getline(timing_tasks, line)){ + if(line.find(" SetTSArray (i") == std::string::npos){ + temp_file << line << "\n"; + continue; + } + for(const auto & param : params){ + if(line.find(param.first) != std::string::npos){ + line = line.replace(35, 7, param.second); + break; + } + } + temp_file << line << "\n"; + } + + timing_tasks.close(); + temp_file.close(); + remove(time_path.c_str()); + rename(temp_path.c_str(), time_path.c_str()); + wait(NULL); + std::cout << "Timing parameters are updated...\n"; + } +} diff --git a/sources/api/prog.h b/sources/api/prog.h new file mode 100644 index 0000000..d9febd7 --- /dev/null +++ b/sources/api/prog.h @@ -0,0 +1,120 @@ +#ifndef PROG_H +#define PROG_H + +#include <vector> +#include <map> +#include <string> +#include "instruction.h" + +class Program{ +private: + std::map<std::string, uint> labels; + std::map<uint, std::string> branches; + std::map<uint, Inst> warnings; + std::vector<Inst> program; + std::vector<Mininst> minprogram; + int spc = 0; + + bool dumpRegsCalled; + + void preprocess_branches(); + void linear_analysis(); + void insert_generated(); + Inst* get_insts(); + +public: + enum class SEQ_TYPE { WRITE, READ }; + enum BR_TYPE { BEQ, BL, JUMP }; + + + Program(); + + /** + * Load an instruction stream from file + * @param fname name of the file + */ + Program(std::string fname); + + void add_mininst(Mininst mi, int wait_after); + void add_wait(int wait_cycles); + void pack_minprogram(); + /** + * Add an instruction to the program + * @param i instruction to add, usually generated by instruction.h functions (e.g. SMC_ADD()) + */ + void add_inst(Inst i); + /** + * Add four mininsts to the program + */ + void add_inst(Mininst,Mininst,Mininst,Mininst); + void add_label(std::string name); + /** + * Add a conditional branch to the program + * @bt the type of branch instruction to be executed: + * either one of: BR_TYPE.BEQ, BR_TYPE.BL or BR_TYPE.JUMP + * @rs1 branch instr. operand 1 + * @rs2 branch instr. operand 2 + * @tgt branch target to jump to if condition is satisfied + */ + void add_branch(BR_TYPE bt, int rs1, int rs2, std::string tgt); + + /** + * Append another program to this program. + * @param p program to append to the end of this program. + */ + void add_below(const Program &p); + + /** + * Parse SMC programs + * @param fname file to read from + */ + void parse_from_file(std::string fname); + + Inst* get_inst_array(); + + /** + * @return size of the program in bytes + */ + int size(); + + /** + * Print instructions in a human readable form + */ + void pretty_print(); + + /** + * Print instruction binaries + */ + void bin_dump(); + /** + * Save this instruction stream to file + * @param fname name of the file + */ + void save_bin(const std::string &fname); + void save_coe_here(const std::string &prj_dir); + void save_coe(const std::string &prj_dir); + void debug(const std::string &prj_dir, bool first); + void debug(const std::string &prj_dir, const std::string &filename); + + /** + * Insert routine required to dump register file content. + * The generated routine requires registers 13-15 to operate correctly. + * It won't do any context-switch and registers 13-15 will have garbage + * values when this routine is executed. It will also use DRAM bank + * 15 row 0-1 to store and readback data, it will also precharge all banks. + * This routine will also mess with WDATA reg content *after* dumping it. + * + * Aside from all the above, you need to intercept reads yourself. Make sure + * that there is no other data left in the buffers before this routine executes + * and call platform::readRegisterDump(). + * You have been warned. + * You have been warned twice. + * You did not hear that wrong, this will dump all register content. + */ + void dump_registers(); + + bool isDumpRegsCalled(); + +}; + +#endif diff --git a/sources/api/pySoftMC/.gitignore b/sources/api/pySoftMC/.gitignore new file mode 100644 index 0000000..f1fe8d1 --- /dev/null +++ b/sources/api/pySoftMC/.gitignore @@ -0,0 +1 @@ +*.so
\ No newline at end of file diff --git a/sources/api/pySoftMC/Makefile b/sources/api/pySoftMC/Makefile new file mode 100644 index 0000000..771df53 --- /dev/null +++ b/sources/api/pySoftMC/Makefile @@ -0,0 +1,36 @@ +# https://stackoverflow.com/questions/4933285/how-to-determine-python-version-in-makefile +python_version_full := $(wordlist 2,4,$(subst ., ,$(shell python3 --version 2>&1))) +python_version := $(word 1,${python_version_full}).$(word 2,${python_version_full}) + +program_NAME := pySoftMC +program_CXX_SRCS := $(wildcard ../*.c) $(wildcard ../*.cpp) ./pySoftMC.cpp +program_CXX_OBJS := ${program_CXX_SRCS:.cpp=.o} +program_CXX_OBJS := ${program_CXX_OBJS:.c=.o} +program_OBJS := $(program_CXX_OBJS) + +program_INCLUDE_DIRS := . .. /usr/include/python${python_version} ../ext/pybind11/include/pybind11 +program_LIBRARY_DIRS := +program_LIBRARIES := python${python_version} +CPPFLAGS += -g -std=c++14 -pthread -O3 -fPIC -Wall + +CPPFLAGS += $(foreach includedir,$(program_INCLUDE_DIRS),-I$(includedir)) -DPYSMC +LDFLAGS += $(foreach librarydir,$(program_LIBRARY_DIRS),-L$(librarydir)) +LDFLAGS += $(foreach library,$(program_LIBRARIES),-l$(library)) + +SHARED_LIB_FLAGS := -shared -fPIC + +CC=g++ + +.PHONY: all clean distclean + +all: $(program_NAME) + @- $(RM) $(program_OBJS) + +$(program_NAME): $(program_OBJS) + $(CC) $(CPPFLAGS) ${SHARED_LIB_FLAGS} $(program_OBJS) -o $(program_NAME)$(shell python3-config --extension-suffix) $(LDFLAGS) + +clean: + @- $(RM) $(program_NAME)$(shell python3-config --extension-suffix) + @- $(RM) $(program_OBJS) + +distclean: clean diff --git a/sources/api/pySoftMC/example.py b/sources/api/pySoftMC/example.py new file mode 100644 index 0000000..2dd2615 --- /dev/null +++ b/sources/api/pySoftMC/example.py @@ -0,0 +1,153 @@ +from pySoftMC import * +import numpy as np + +# Initialize SoftMC platform +platform = SoftMCPlatform() +err = platform.init() +if (err != 0): + print("SoftMC platform initialization failed!") + exit(-1) +platform.reset_fpga() + +# Define registers +CASR = 0 +BASR = 1 +RASR = 2 +BAR = 3 +RAR = 4 +CAR = 5 +PATTERN_REG = 6 + + +def initialize_row(platform: SoftMCPlatform, data32: int, bank: int, row: int): + p = Program() + p.add_inst(SMC_LI(bank, BAR)) + p.add_inst(SMC_LI(row, RAR)) + p.add_inst(SMC_LI(8, CASR)) + + p.add_inst(SMC_LI(data32, PATTERN_REG)) + for i in range(16): + p.add_inst(SMC_LDWD(PATTERN_REG, i)) + + p.add_inst(SMC_PRE(BAR, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_LI(0, CAR)) + p.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()) + + p.add_inst(SMC_ACT(BAR, 0, RAR, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()) + + for i in range(128): + p.add_inst(SMC_WRITE(BAR, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_SLEEP(3)) + + p.add_inst(SMC_PRE(BAR, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()) + + p.add_inst(SMC_END()) + platform.execute(p) + +def read_row(platform: SoftMCPlatform, bank: int, row: int): + p = Program() + p.add_inst(SMC_LI(bank, BAR)) + p.add_inst(SMC_LI(row, RAR)) + p.add_inst(SMC_LI(8, CASR)) + + p.add_inst(SMC_PRE(BAR, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_LI(0, CAR)) + p.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()) + + p.add_inst(SMC_ACT(BAR, 0, RAR, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()) + + for i in range(128): + p.add_inst(SMC_READ(BAR, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_SLEEP(3)) + + p.add_inst(SMC_PRE(BAR, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()) + + p.add_inst(SMC_END()) + platform.execute(p) + +def hammer_row_double(bank, aggressor_row_1, aggressor_row_2, hammer_count, RAS_scale, RP_scale): + HMR_COUNTER_REG = 7 + NUM_HMR_REG = 8 + + p = Program() + p.add_inst(SMC_LI(bank, BAR)) + p.add_inst(SMC_LI(aggressor_row_1, RAR)) + + p.add_inst(SMC_LI(0, HMR_COUNTER_REG)) + p.add_inst(SMC_LI(hammer_count, NUM_HMR_REG)) + + p.add_label("HMR_BEGIN") + if (RAS_scale > 1): + p.add_inst(SMC_SLEEP(5 * (RAS_scale - 1))) + + p.add_inst(SMC_PRE(BAR, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_LI(aggressor_row_1, RAR)) + if (RP_scale > 1): + p.add_inst(SMC_SLEEP(3 * (RP_scale - 1))) + + + p.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_ACT(BAR, 0, RAR, 0)) + p.add_inst(SMC_LI(aggressor_row_2, RAR)) + p.add_inst(SMC_SLEEP(5)) + if (RAS_scale > 1): + p.add_inst(SMC_SLEEP(5 * (RAS_scale - 1))) + + p.add_inst(SMC_PRE(BAR, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_ADDI(HMR_COUNTER_REG, 1, HMR_COUNTER_REG)) + if (RP_scale > 1): + p.add_inst(SMC_SLEEP(3 * (RP_scale - 1))) + + + p.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_ACT(BAR, 0, RAR, 0)) + p.add_branch(Program.BR_TYPE.BL, HMR_COUNTER_REG, NUM_HMR_REG, "HMR_BEGIN") + + + p.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()) + p.add_inst(SMC_END()) + + platform.execute(p) + +def check_flips(buffer, pattern): + d = [] + out = np.bitwise_xor(buffer, pattern) + for cl in range(128): + for w in range(8): + if (out[cl * 8 + w]) != 0: + _w = out[cl * 8 + w] + for byte in range(8): + for bit in range(8): + bit_id = np.uint64(byte * 8 + bit) + if ((np.right_shift(_w, bit_id)) & np.uint64(0x1)): + d.append((i, cl, w, byte, bit)) + print(f"ITR {i}\tCL {cl}\t Word {w}\t Byte {byte}\t Bit {bit}") + + +# Get the buffer from the underlying C++ API +buffer = np.frombuffer(platform.get_buffer_memoryview()[:8192], dtype=np.uint64) + +ITR_COUNT=10 +ACT_COUNT = 100000 +pattern = 0xFFFFFFFF + +for i in range(ITR_COUNT): + initialize_row(platform, 0xFFFFFFFF, 0, 233) + initialize_row(platform, 0x0, 0, 234) + initialize_row(platform, 0xFFFFFFFF, 0, 235) + + hammer_row_double(0, 233, 235, ACT_COUNT, 1, 1) + + read_row(platform, 0, 234) + platform.receiveData(8*1024) + + check_flips(buffer, 0x0) + diff --git a/sources/api/pySoftMC/pySoftMC.cpp b/sources/api/pySoftMC/pySoftMC.cpp new file mode 100644 index 0000000..992d9a5 --- /dev/null +++ b/sources/api/pySoftMC/pySoftMC.cpp @@ -0,0 +1,95 @@ +#include "platform.h" +#include "instruction.h" +#include "prog.h" + +#include "ext/pybind11/include/pybind11/pybind11.h" +namespace py = pybind11; + +PYBIND11_MODULE(pySoftMC, m) +{ + py::enum_<PC_TYPE>(m, "PC_TYPE") + .value("WRITE", PC_TYPE::WRITE) + .value("READ", PC_TYPE::READ) + .value("PRE", PC_TYPE::PRE) + .value("ACT", PC_TYPE::ACT) + .value("ZQ", PC_TYPE::ZQ) + .value("REF", PC_TYPE::REF) + .value("CYC", PC_TYPE::CYC) + .export_values(); + + m.def("SMC_LD", &SMC_LD, py::arg("rb"), py::arg("offset"), py::arg("rt")); + m.def("SMC_ST", &SMC_ST, py::arg("rb"), py::arg("offset"), py::arg("rt")); + + m.def("SMC_AND", &SMC_AND, py::arg("rs1"), py::arg("rs2"), py::arg("rt")); + m.def("SMC_OR", &SMC_OR, py::arg("rs1"), py::arg("rs2"), py::arg("rt")); + m.def("SMC_XOR", &SMC_XOR, py::arg("rs1"), py::arg("rs2"), py::arg("rt")); + m.def("SMC_ADD", &SMC_ADD, py::arg("rs1"), py::arg("rs2"), py::arg("rt")); + m.def("SMC_ADDI", &SMC_ADDI, py::arg("rs1"), py::arg("imd"), py::arg("rt")); + m.def("SMC_SUB", &SMC_SUB, py::arg("rs1"), py::arg("rs2"), py::arg("rt")); + m.def("SMC_SUBI", &SMC_SUBI, py::arg("rs1"), py::arg("imd"), py::arg("rt")); + m.def("SMC_SRC", &SMC_SRC, py::arg("rs1"), py::arg("rt")); + + m.def("SMC_LI", &SMC_LI, py::arg("imd"), py::arg("rt")); + m.def("SMC_MV", &SMC_MV, py::arg("rs1"), py::arg("rt")); + + m.def("SMC_LDWD", &SMC_LDWD, py::arg("rs1"), py::arg("offset")); + m.def("SMC_LDPC", &SMC_LDPC, py::arg("type"), py::arg("rt")); + + m.def("SMC_BL", &SMC_BL, py::arg("rs1"), py::arg("rs2"), py::arg("tgt")); + m.def("SMC_BEQ", &SMC_BEQ, py::arg("rs1"), py::arg("rs2"), py::arg("tgt")); + m.def("SMC_JUMP", &SMC_JUMP, py::arg("tgt")); + + m.def("SMC_END", &SMC_END); + m.def("SMC_INFO", &SMC_INFO, py::arg("rdcnt")); + m.def("SMC_SLEEP", &SMC_SLEEP, py::arg("samt")); + + m.def("SMC_WRITE", &SMC_WRITE, py::arg("bar"), py::arg("ibar"), py::arg("car"), py::arg("icar"), py::arg("BL4"), py::arg("ap")); + m.def("SMC_READ", &SMC_READ, py::arg("bar"), py::arg("ibar"), py::arg("car"), py::arg("icar"), py::arg("BL4"), py::arg("ap")); + m.def("SMC_PRE", &SMC_PRE, py::arg("bar"), py::arg("ibar"), py::arg("pall")); + m.def("SMC_ACT", &SMC_ACT, py::arg("bar"), py::arg("ibar"), py::arg("rar"), py::arg("irar")); + + m.def("SMC_ZQ", &SMC_ZQ); + m.def("SMC_REF", &SMC_REF); + m.def("SMC_NOP", &SMC_NOP); + m.def("SMC_SRE", &SMC_SRE); + m.def("SMC_SRX", &SMC_SRX); + + m.def("__pack_mininsts", &__pack_mininsts, py::arg("i1"), py::arg("i2"), py::arg("i3"), py::arg("i4")); + + py::class_<SoftMCPlatform>(m, "SoftMCPlatform") + .def(py::init()) + .def("init", &SoftMCPlatform::init) + .def("reset_fpga", &SoftMCPlatform::reset_fpga) + .def("execute", &SoftMCPlatform::execute, py::arg("program")) + .def("receiveData", &SoftMCPlatform::py_receiveData, py::arg("num_words")) + .def("get_buffer_memoryview", &SoftMCPlatform::get_buffer_memoryview) + .def("set_aref", &SoftMCPlatform::set_aref, py::arg("on")) + .def("readRegisterDump", &SoftMCPlatform::readRegisterDump) + .def("count_bitflips_in_row", &SoftMCPlatform::count_bitflips_in_row, py::arg("char_data")); + + py::class_<Program> program(m, "Program"); + + py::enum_<Program::SEQ_TYPE>(program, "SEQ_TYPE") + .value("WRITE", Program::SEQ_TYPE::WRITE) + .value("READ", Program::SEQ_TYPE::READ) + .export_values(); + + py::enum_<Program::BR_TYPE>(program, "BR_TYPE") + .value("BEQ", Program::BR_TYPE::BEQ) + .value("BL", Program::BR_TYPE::BL) + .value("JUMP", Program::BR_TYPE::JUMP) + .export_values(); + + program + .def(py::init()) + .def("add_mininst", &Program::add_mininst, py::arg("mi"), py::arg("wait_after")) + .def("add_wait", &Program::add_wait, py::arg("wait_cycles")) + .def("pack_minprogram", &Program::pack_minprogram) + .def("add_inst", py::overload_cast<Inst>(&Program::add_inst), py::arg("inst")) + .def("add_inst", py::overload_cast<Mininst, Mininst, Mininst, Mininst>(&Program::add_inst)) + .def("add_label", &Program::add_label, py::arg("name")) + .def("add_branch", &Program::add_branch, py::arg("type"), py::arg("rs1"), py::arg("rs2"), py::arg("tgt")) + .def("add_below", &Program::add_below) + .def("pretty_print", &Program::pretty_print) + .def("dump_registers", &Program::dump_registers); +} diff --git a/sources/apps/.gitignore b/sources/apps/.gitignore new file mode 100644 index 0000000..5761abc --- /dev/null +++ b/sources/apps/.gitignore @@ -0,0 +1 @@ +*.o diff --git a/sources/apps/DebugExample/Makefile b/sources/apps/DebugExample/Makefile new file mode 100755 index 0000000..13ecbb0 --- /dev/null +++ b/sources/apps/DebugExample/Makefile @@ -0,0 +1,32 @@ +program_NAME := Debug_example +program_CXX_SRCS := debug_example.cpp $(wildcard ../../api/*.c) $(wildcard ../../api/*.cpp) +program_CXX_OBJS := ${program_CXX_SRCS:.cpp=.o} +program_CXX_OBJS := ${program_CXX_OBJS:.c=.o} +program_OBJS := $(program_CXX_OBJS) +program_INCLUDE_DIRS := ../../api ../../../boost-lib +program_LIBRARY_DIRS := +program_LIBRARIES := +CPPFLAGS += -g -std=c++11 -pthread -O3 + +CPPFLAGS += $(foreach includedir,$(program_INCLUDE_DIRS),-I$(includedir)) +LDFLAGS += $(foreach librarydir,$(program_LIBRARY_DIRS),-L$(librarydir)) +LDFLAGS += $(foreach library,$(program_LIBRARIES),-l$(library)) + +CC=g++ + +.PHONY: all clean distclean + +all: $(program_NAME) + +$(program_NAME): $(program_OBJS) + $(CC) $(CPPFLAGS) $(program_OBJS) -o $(program_NAME) $(LDFLAGS) + +clean: + @- $(RM) $(program_NAME) + @- $(RM) $(program_OBJS) + +parser: + $(MAKE) -C ../../api/lexyacc + cp ../../api/lexyacc/smc_parser . + +distclean: clean diff --git a/sources/apps/DebugExample/debug_example.cpp b/sources/apps/DebugExample/debug_example.cpp new file mode 100644 index 0000000..6581354 --- /dev/null +++ b/sources/apps/DebugExample/debug_example.cpp @@ -0,0 +1,58 @@ +#include "instruction.h" +#include "prog.h" +#include "platform.h" +#include <fstream> +#include <iostream> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <cstring> +#include <list> + +using namespace std; + +#define PROJECT_DIRECTORY "<YOUR PROJECT DIRECTORY>" + +#define ADD_REG 7 +#define LD_REG 8 +#define STR_REG 9 + +/** + * @return an instruction formed by NOPs + */ +Inst all_nops() +{ + return __pack_mininsts(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()); +} + +int main() +{ + + Program program; + + program.add_inst(SMC_LI(5, 3)); //R3=5 + program.add_inst(SMC_LI(15, 4)); //R4=15 + program.add_label("LOOP"); + program.add_inst(SMC_ADDI(3, 1, 3)); //R3++ + //If R4>R3, go to LOOP + program.add_branch(program.BR_TYPE::BL,3,4, "LOOP"); + program.add_inst(all_nops()); //R3=R4=15 + program.add_inst(SMC_LI(121, LD_REG)); + program.add_inst(SMC_LI(62, STR_REG)); + program.add_inst(SMC_LI(100, ADD_REG)); + program.add_inst(SMC_ST(ADD_REG, 0, STR_REG)); + program.add_inst(all_nops()); + program.add_inst(SMC_LD(ADD_REG, 0, LD_REG)); + program.add_inst(all_nops()); + program.add_inst(SMC_END()); + + //DRAM timing parameters are updated with timings.txt + program.debug(PROJECT_DIRECTORY, "timings.txt"); + + //Load the program to instruction memory + program.save_coe(PROJECT_DIRECTORY); + program.pretty_print(); + + //Start the debugger + program.debug(PROJECT_DIRECTORY, false); +} diff --git a/sources/apps/DebugExample/timings.txt b/sources/apps/DebugExample/timings.txt new file mode 100644 index 0000000..8de3bed --- /dev/null +++ b/sources/apps/DebugExample/timings.txt @@ -0,0 +1,14 @@ +tRTP = 7500 +tWTRc_S = 2 +tWTRc_L = 5 +tWR = 15000 +tMOD = 36000 +tRCD = 16000 +tRC = 55000 +tRP = 15000 +tCCDc_S = 4 +tCCDc_L = 5 +tRAS = 39000 +tRRDc_S = 4 +tRRDc_L = 4 +tFAW = 20000 diff --git a/sources/apps/QUAC-TRNG/.gitignore b/sources/apps/QUAC-TRNG/.gitignore new file mode 100644 index 0000000..8fce603 --- /dev/null +++ b/sources/apps/QUAC-TRNG/.gitignore @@ -0,0 +1 @@ +data/ diff --git a/sources/apps/QUAC-TRNG/Makefile.ent b/sources/apps/QUAC-TRNG/Makefile.ent new file mode 100644 index 0000000..468a10b --- /dev/null +++ b/sources/apps/QUAC-TRNG/Makefile.ent @@ -0,0 +1,28 @@ +program_NAME := bin/ENT +program_CXX_SRCS := src/slow_entropy.cpp src/util.cpp $(wildcard ../../api/*.c) $(wildcard ../../api/*.cpp) +program_CXX_OBJS := ${program_CXX_SRCS:.cpp=.o} +program_CXX_OBJS := ${program_CXX_OBJS:.c=.o} +program_OBJS := $(program_CXX_OBJS) +program_INCLUDE_DIRS := ../../api ../../../boost-lib +program_LIBRARY_DIRS := +program_LIBRARIES := +CPPFLAGS += -g -std=c++11 -pthread -O3 + +CPPFLAGS += $(foreach includedir,$(program_INCLUDE_DIRS),-I$(includedir)) +LDFLAGS += $(foreach librarydir,$(program_LIBRARY_DIRS),-L$(librarydir)) +LDFLAGS += $(foreach library,$(program_LIBRARIES),-l$(library)) + +CC=g++ + +.PHONY: all clean distclean + +all: $(program_NAME) + +$(program_NAME): $(program_OBJS) + $(CC) $(CPPFLAGS) $(program_OBJS) -o $(program_NAME) $(LDFLAGS) + +clean: + @- $(RM) $(program_NAME) + @- $(RM) $(program_OBJS) + +distclean: clean diff --git a/sources/apps/QUAC-TRNG/Makefile.observe b/sources/apps/QUAC-TRNG/Makefile.observe new file mode 100644 index 0000000..d5b1adb --- /dev/null +++ b/sources/apps/QUAC-TRNG/Makefile.observe @@ -0,0 +1,32 @@ +program_NAME := bin/OBSERVE +program_CXX_SRCS := src/observe.cpp $(wildcard ../../api/*.c) $(wildcard ../../api/*.cpp) +program_CXX_OBJS := ${program_CXX_SRCS:.cpp=.o} +program_CXX_OBJS := ${program_CXX_OBJS:.c=.o} +program_OBJS := $(program_CXX_OBJS) +program_INCLUDE_DIRS := ../../api ../../../boost-lib +program_LIBRARY_DIRS := +program_LIBRARIES := +CPPFLAGS += -g -std=c++11 -pthread -O3 + +CPPFLAGS += $(foreach includedir,$(program_INCLUDE_DIRS),-I$(includedir)) +LDFLAGS += $(foreach librarydir,$(program_LIBRARY_DIRS),-L$(librarydir)) +LDFLAGS += $(foreach library,$(program_LIBRARIES),-l$(library)) + +CC=g++ + +.PHONY: all clean distclean + +all: $(program_NAME) + +$(program_NAME): $(program_OBJS) + $(CC) $(CPPFLAGS) $(program_OBJS) -o $(program_NAME) $(LDFLAGS) + +clean: + @- $(RM) $(program_NAME) + @- $(RM) $(program_OBJS) + +parser: + $(MAKE) -C ../../api/lexyacc + cp ../../api/lexyacc/smc_parser . + +distclean: clean diff --git a/sources/apps/QUAC-TRNG/Makefile.sha b/sources/apps/QUAC-TRNG/Makefile.sha new file mode 100644 index 0000000..bd0c463 --- /dev/null +++ b/sources/apps/QUAC-TRNG/Makefile.sha @@ -0,0 +1,28 @@ +program_NAME := bin/SHA +program_CXX_SRCS := src/slow_sha.cpp src/util.cpp $(wildcard ../../api/*.c) $(wildcard ../../api/*.cpp) +program_CXX_OBJS := ${program_CXX_SRCS:.cpp=.o} +program_CXX_OBJS := ${program_CXX_OBJS:.c=.o} +program_OBJS := $(program_CXX_OBJS) +program_INCLUDE_DIRS := ../../api ../../../boost-lib +program_LIBRARY_DIRS := +program_LIBRARIES := ssl crypto +CPPFLAGS += -g -std=c++11 -pthread -O3 + +CPPFLAGS += $(foreach includedir,$(program_INCLUDE_DIRS),-I$(includedir)) +LDFLAGS += $(foreach librarydir,$(program_LIBRARY_DIRS),-L$(librarydir)) +LDFLAGS += $(foreach library,$(program_LIBRARIES),-l$(library)) + +CC=g++ + +.PHONY: all clean distclean + +all: $(program_NAME) + +$(program_NAME): $(program_OBJS) + $(CC) $(CPPFLAGS) $(program_OBJS) -o $(program_NAME) $(LDFLAGS) + +clean: + @- $(RM) $(program_NAME) + @- $(RM) $(program_OBJS) + +distclean: clean diff --git a/sources/apps/QUAC-TRNG/README.md b/sources/apps/QUAC-TRNG/README.md new file mode 100644 index 0000000..437d0dd --- /dev/null +++ b/sources/apps/QUAC-TRNG/README.md @@ -0,0 +1,58 @@ +## Scripts to Reproduce QUAC-TRNG + +This directory provides all necessary files and instructions to reproduce the results of [QUAC-TRNG](https://people.inf.ethz.ch/omutlu/pub/QUAC-TRNG-DRAM_isca21.pdf). + +``` +@inproceedings{olgun2022quactrng, + title={QUAC-TRNG: High-Throughput True Random Number Generation Using Quadruple Row Activation in Commodity DRAM Chips}, + author={Ataberk Olgun, Minesh Patel, A. Giray Yaglikci, Haocong Luo, Jeremie S. Kim, F. Nisa Bostanci, Nandita Vijaykumar, Oguz Ergin, and Onur Mutlu}, + year={2021}, + booktitle={ISCA} +} +``` + +### Obtain NIST STS + +Download the NIST standard test suite (NIST STS) for random number generators using [this link](https://github.com/arcetri/sts). Install it under `tools` using the instructions provided in the linked repository's README. + +After installing NIST STS, `QUAC-TRNG/tools/sts` should point to the NIST STS executable. + +### Copy SoftMC_reset to bin/ + +Run the following command to copy the reset program under bin/ directory. + +`cp ../ResetBoard/SoftMC_reset bin/` + +### Entropy Tests + +To run entropy tests, simply execute the `run_ent.py` script (you have to execute this in QUAC-TRNG directory): + +``` +make -f Makefile.ent +python3 scripts/run_ent.py <DIMM_NAME> <TEMPERATURE_IN_CELCIUS> <QUAC_ITERATIONS> <ROW_STRIDE> +``` + +Example: `python3 scripts/run_ent.py hytt04 50 1000 4` + +This script will output one text file for every tested 4-bit data pattern (variable named placement in run_ent.py). Each line in the text file starts with the starting `row id` of a DRAM segment, followed by the observed entropy numbers for each bit in the segment. + +### von Neumann Corrected Bitstreams + +This will collect 1 Mb bitstreams from frequently switching bitlines and input them to NIST STS. The script will stop the moment it finds a cell that passes all NIST tests. + +``` +make -f Makefile.observe +python3 scripts/run_vn_nist.py <DIMM_NAME> <TEMPERATURE_IN_CELCIUS> +``` + +Example: `python3 scripts/run_vn_nist.py hytt04 50` + +### SHA-256 Bitstreams + +This section describes how to get the NIST STS results for SHA-256 processed bitstreams. Doing so consists of two steps: 1) finding the DRAM segments with top 10 entropy, 2) generating bitstreams from the top-10-entropy segments and processing them using SHA-256 and performing NIST STS tests. + +First, run `make -f Makefile.sha`. + +1- Obtain the entropy distribution to find the desired 4-bit data pattern (refer to "Entropy Tests"). +2- Get the segments with max. entropy: `python3 scripts/top_ents.py <DIMM_NAME> <PATTERN> <TEMPERATURE>` +3- Run the SHA-256 bitstream generator script: `python3 scripts/sha_bitstreams.py <DIMM_NAME> <PATTERN> <TEMPERATURE>`
\ No newline at end of file diff --git a/sources/apps/QUAC-TRNG/bin/.gitkeep b/sources/apps/QUAC-TRNG/bin/.gitkeep new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/sources/apps/QUAC-TRNG/bin/.gitkeep diff --git a/sources/apps/QUAC-TRNG/scripts/convert_ent.py b/sources/apps/QUAC-TRNG/scripts/convert_ent.py new file mode 100644 index 0000000..b2c66ea --- /dev/null +++ b/sources/apps/QUAC-TRNG/scripts/convert_ent.py @@ -0,0 +1,32 @@ +import os +import argparse +from numpy import array + +parser = argparse.ArgumentParser(description = "Run this script to preprocess entropy files") +parser.add_argument('indir', help="Path to entropy files") +args = parser.parse_args() + +files = [] +for fname in os.listdir(args.indir): + files.append(fname) + +for file in files: + print("Processing:", file) + # Add this placement + fpath = os.path.normpath(args.indir + '/' + file) + infile = open(fpath,'r') + wfile = file.split('.')[0] + '_floats.bin' + wfpath = os.path.normpath(args.indir + '/' + wfile) + output_file = open(wfpath, 'wb') + + lines = infile.readlines() + + # First row reserved for metadata + for lidx in range(1,len(lines)): + line = lines[lidx].strip() + split = line.split() + elements = [float(i) for i in split[1:]] + a = array(elements,'float32') + a.tofile(output_file) + + output_file.close()
\ No newline at end of file diff --git a/sources/apps/QUAC-TRNG/scripts/ft200_rc/README.md b/sources/apps/QUAC-TRNG/scripts/ft200_rc/README.md new file mode 100644 index 0000000..de91b27 --- /dev/null +++ b/sources/apps/QUAC-TRNG/scripts/ft200_rc/README.md @@ -0,0 +1,21 @@ +# FT200 remote read and write + +Make sure that the FT200 controller match the parameters you are using in your script. +The relevant parameters are: + +PASS-0101 menu (see documentation in doc folder): +* ldN0 (Device address) +* bAUd (rate) +* UCR (parity) + +## Requirements +* pip3 install -U minimalmodbus + + +## Add your user to the dialout group + +sudo gpasswd --add ${USER} dialout +(You need to log out and log back in again for it to be effective) + +## Execute test +* python3 ft200.py diff --git a/sources/apps/QUAC-TRNG/scripts/ft200_rc/__init__.py b/sources/apps/QUAC-TRNG/scripts/ft200_rc/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/sources/apps/QUAC-TRNG/scripts/ft200_rc/__init__.py diff --git a/sources/apps/QUAC-TRNG/scripts/ft200_rc/docs/1189902393_ft200_lcd.pdf b/sources/apps/QUAC-TRNG/scripts/ft200_rc/docs/1189902393_ft200_lcd.pdf Binary files differnew file mode 100644 index 0000000..8c18eaa --- /dev/null +++ b/sources/apps/QUAC-TRNG/scripts/ft200_rc/docs/1189902393_ft200_lcd.pdf diff --git a/sources/apps/QUAC-TRNG/scripts/ft200_rc/docs/210.pdf b/sources/apps/QUAC-TRNG/scripts/ft200_rc/docs/210.pdf Binary files differnew file mode 100644 index 0000000..e4d0d39 --- /dev/null +++ b/sources/apps/QUAC-TRNG/scripts/ft200_rc/docs/210.pdf diff --git a/sources/apps/QUAC-TRNG/scripts/ft200_rc/ft200.py b/sources/apps/QUAC-TRNG/scripts/ft200_rc/ft200.py new file mode 100644 index 0000000..aa50a1f --- /dev/null +++ b/sources/apps/QUAC-TRNG/scripts/ft200_rc/ft200.py @@ -0,0 +1,182 @@ +""""" +Code for monitoring and controlling temperature in the Maxwell FT200 controller using the Modbus RS485 protocol +@author: Lois Orosa +""""" +import minimalmodbus as mm, serial, time +import time +import os +import sys +import pathlib + +class FT200: + """ + Controls and monitors the Maxwell FT200 tempoerature controller + """ + READ_ERROR = -999 + WRITE_ERROR = -998 + def __init__(self, _baud=9600, _ft200_addr=1, _temp_tolerance=2): + self.baud = _baud + self.write_reg = 5 # Write register + self.read_reg = 0 # Read register + self.ft200_addr = _ft200_addr # Address of the device (by default is 1) + self.temp_tolerance = _temp_tolerance # Temperature tolerance. By default, we consider that we reach the target value if the measured value is within +-0.2C + + try: + self.device = self.findDevice() # Get the USB-RS485 device + print("USB-RS485 Device: "+str(self.device)) + self.instrument = mm.Instrument(self.device , 3 , debug = False) + self.instrument.serial.baudrate = self.baud + self.instrument.address = self.ft200_addr + self.instrument.serial.timeout = 1 + self.instrument.serial.parity = serial.PARITY_NONE + self.instrument.mode = mm.MODE_RTU + print("Device "+self.device+": conected") + except: + raise + + # Automatically find the USB-RS485 Device + def findDevice(self): + ''' + Find the USB-RS485 device + This function might raise two exceptions: + -> NameError: There is more than 1 USB-RS485 device, or other USB device has a similar name as the USB-RS485 device + -> IOError: Device not found. + ''' + DEVICENAME = "1a86_USB" + name = '' + while True: + path = pathlib.Path(__file__).parent.absolute() + f = os.popen(str(path)+"/usb_dev_path.sh") + for device in f: + if DEVICENAME in device: + if name == '': + name = device.split()[0] + else: + raise NameError("There are at least two USB devices whose name contains "+str(DEVICENAME)) + if name == '': + raise IOError("Device not found.") + return name + + # Read Temperature from FT200 + # The output value has 1 decimal, and it is multiplied by 10 + # E.g., data= 102 represents a temperature of 10.2C + def readTemp(self): + try: + data = self.instrument.read_register(self.read_reg, functioncode=3) + return data + except IOError: + print("Failed to read from instrument") + return self.READ_ERROR + + # Check if the temperature reached a value + # The input value has 1 decimal, and it is multiplied by 10 + # E.g., data= 102 represents a temperature of 10.2C + def isTemp(self, temperature): + try: + data = self.instrument.read_register(self.read_reg, functioncode=3) + if data >= (temperature-self.temp_tolerance) and data <= (temperature+self.temp_tolerance): + return True + else: + return False + except IOError: + print("Failed to read from instrument") + return False + + # Set Temperature FT200 + # The value should be a integer, and it should be the temperature value with 1 decimal multiplied by 10 + # E.g., value= 102 represents a temperature of 10.2C + def setTemp(self, value): + try: + self.instrument.write_register(self.write_reg, value, functioncode=6) + except IOError: + print("Failed to write to instrument") + return self.WRITE_ERROR + return 0 + + # Set the temperature and wait for the temperature to be stable + # Optimized version that reach the target temperature earlier + # The value should be a integer, and it should be the temperature value with 1 decimal multiplied by 10 + # E.g., value= 102 represents a temperature of 10.2C + def autoSetAndWait(self, temperature, debug = False): + acceleration_factor = 5 + stability = 10 # Iterations that the temperature needs to be stable + time_sleep = 1 # 1 second wait per loop iteration + stable_count = stability # Counter for taking into account the stability of the temperature + stable_margin = 20 # We do not change the set temperature if we are within 2C from the target + optTemp = temperature + self.setTemp(optTemp) # Set the temperature + curTemp = self.readTemp() # Read the temperature + if debug: + print("[TEMPERATURE] current: "+str(curTemp/10.0)+", goal: "+str(temperature/10.0)+", programmed: "+str(optTemp/10.0)) + while (((curTemp > (temperature + self.temp_tolerance)) or (curTemp < (temperature - self.temp_tolerance))) or (stable_count!=0)): + if debug: + print("[TEMPERATURE] current: "+str(curTemp/10.0)+", goal: "+str(temperature/10.0)+", programmed: "+str(optTemp/10.0)+", stable_count: "+str(stable_count)) + prev_optTemp = optTemp + if (temperature - curTemp) > 0: #(stable_margin): # If we are far away more than 2C + optTemp = temperature + (temperature - curTemp)*acceleration_factor # Optimization to accelerate convergence + else: + optTemp = temperature + if optTemp != prev_optTemp: + self.setTemp(optTemp) # Set temperature only if different from the previous iteration + + time.sleep(time_sleep) # WAit for a while + + curTemp = self.readTemp() # Read the temperature + while (curTemp == self.READ_ERROR): # If there is an error, read again + curTemp = self.readTemp() + # Five iterations in the range + if (curTemp > (temperature + self.temp_tolerance)) or (curTemp < (temperature - self.temp_tolerance)): + stable_count = stability + else: + stable_count -= 1 # The temperature has to remain stable for a while + + def autoSetAndWait_2(self, temperature, debug = False): + acceleration_factor = 150 + stability = 10 # Iterations that the temperature needs to be stable + time_sleep = 1 # 1 second wait per loop iteration + stable_count = stability # Counter for taking into account the stability of the temperature + stable_margin = 5 # We do not change the set temperature if we are within 2C from the target + optTemp = temperature + self.setTemp(optTemp) # Set the temperature + curTemp = self.readTemp() # Read the temperature + if debug: + print("[TEMPERATURE] current: "+str(curTemp/10.0)+", goal: "+str(temperature/10.0)+", programmed: "+str(optTemp/10.0)) + while (((curTemp > (temperature + self.temp_tolerance)) or (curTemp < (temperature - self.temp_tolerance))) or (stable_count!=0)): + if debug: + print("[TEMPERATURE] current: "+str(curTemp/10.0)+", goal: "+str(temperature/10.0)+", programmed: "+str(optTemp/10.0)+", stable_count: "+str(stable_count)) + prev_optTemp = optTemp + if abs(temperature - curTemp) > (stable_margin): # If we are far away more than 2C + if temperature > curTemp: + optTemp = temperature + acceleration_factor # Optimization to accelerate convergence + else: + optTemp = temperature - acceleration_factor # Optimization to accelerate convergence + else: + optTemp = temperature + + time.sleep(time_sleep) # WAit for a while + + curTemp = self.readTemp() # Read the temperature + while (curTemp == self.READ_ERROR): # If there is an error, read again + curTemp = self.readTemp() + # Five iterations in the range + if (curTemp > (temperature + self.temp_tolerance)) or (curTemp < (temperature - self.temp_tolerance)): + stable_count = stability + else: + stable_count -= 1 # The temperature has to remain stable for a while + +if __name__ == '__main__': + ## Example of how to use the FT200 class + try: + tc = FT200() + except Exception as error: + print("[ERROR] "+ repr(error)) + sys.exit(0) + + # Temperature + value = int(sys.argv[1]) * 10 + print(value) + print(tc.readTemp()) + tc.autoSetAndWait(value,True) + + + diff --git a/sources/apps/QUAC-TRNG/scripts/ft200_rc/usb_dev_path.sh b/sources/apps/QUAC-TRNG/scripts/ft200_rc/usb_dev_path.sh new file mode 100755 index 0000000..241aef1 --- /dev/null +++ b/sources/apps/QUAC-TRNG/scripts/ft200_rc/usb_dev_path.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# https://unix.stackexchange.com/questions/144029/command-to-determine-ports-of-a-device-like-dev-ttyusb0 +echo "USB devices:" +for sysdevpath in $(find /sys/bus/usb/devices/usb*/ -name dev); do + ( + syspath="${sysdevpath%/dev}" + devname="$(udevadm info -q name -p $syspath)" +[[ "$devname" == "bus/"* ]] && exit + eval "$(udevadm info -q property --export -p $syspath)" +[[ -z "$ID_SERIAL" ]] && exit + echo "/dev/$devname - $ID_SERIAL" + ) +done diff --git a/sources/apps/QUAC-TRNG/scripts/run_ent.py b/sources/apps/QUAC-TRNG/scripts/run_ent.py new file mode 100644 index 0000000..5163c5f --- /dev/null +++ b/sources/apps/QUAC-TRNG/scripts/run_ent.py @@ -0,0 +1,43 @@ +import os +import argparse +import subprocess +import time + +parser = argparse.ArgumentParser(description = "Collects entropy for different data patterns") + +parser.add_argument('dimm', help="DIMM label under test") +parser.add_argument('temperature', help="DIMM temperature in centigrades") +parser.add_argument('iters', help="How many times do we QUAC to find entropy") +parser.add_argument('stride', help="Stride (in rows) used to skim through segments, default: 4") + +args = parser.parse_args() + +# ---------------- SETUP DIRECTORIES ------------------- + +if not os.path.isdir("data"): + os.mkdir("data") + +if not os.path.isdir("data/" + args.dimm): + os.mkdir("data/" + args.dimm) + +if not os.path.isdir("data/" + args.dimm + "/ent"): + os.mkdir("data/" + args.dimm + "/ent") + +if not os.path.isdir("data/" + args.dimm + "/ent/" + args.temperature + "C"): + os.mkdir("data/" + args.dimm + "/ent/" + args.temperature + "C") + +outdir = os.path.normpath("data/" + args.dimm + "/ent/" + args.temperature + "C") + +# -------------------- RUN TESTS ----------------------- + +placements = ["1000", "1001", "1010", "1011", "1100" , "1101", "1110", "1111", \ + "0111", "0110", "0101", "0100", "0011" , "0010", "0001", "0000"] + +ent_path = os.path.normpath("bin/ENT") +results_path = outdir +stride = args.stride +iters = args.iters + +for placement in placements: + print("sudo " + ent_path + " " + results_path + " " + placement + " " + stride + " " + iters) + os.system("sudo " + ent_path + " " + results_path + " " + placement + " " + stride + " " + iters) diff --git a/sources/apps/QUAC-TRNG/scripts/run_vn_nist.py b/sources/apps/QUAC-TRNG/scripts/run_vn_nist.py new file mode 100644 index 0000000..c439da8 --- /dev/null +++ b/sources/apps/QUAC-TRNG/scripts/run_vn_nist.py @@ -0,0 +1,125 @@ +import os +import argparse +import subprocess +import time + +parser = argparse.ArgumentParser(description = "Run this script to evaluate bitstreams") + +parser.add_argument('dimm', help="DIMM label under test") +parser.add_argument('temperature', help="DIMM temperature in centigrades") + +args = parser.parse_args() + +# ---------------- SETUP DIRECTORIES ------------------- + +if not os.path.isdir("data"): + os.mkdir("data") + +if not os.path.isdir("data/" + args.dimm): + os.mkdir("data/" + args.dimm) + +if not os.path.isdir("data/" + args.dimm + "/vnc"): + os.mkdir("data/" + args.dimm + "/vnc") + +if not os.path.isdir("data/" + args.dimm + "/vnc/" + args.temperature + "C"): + os.mkdir("data/" + args.dimm + "/vnc/" + args.temperature + "C") + +if not os.path.isdir("data/" + args.dimm + "/vnc/" + + args.temperature + "C/nist_results"): + os.mkdir("data/" + args.dimm + "/vnc/" + args.temperature + "C/nist_results") + +if not os.path.isdir("data/" + args.dimm + "/vnc/" + + args.temperature + "C/binaries"): + os.mkdir("data/" + args.dimm + "/vnc/" + args.temperature + "C/binaries") + +outdir = os.path.normpath("data/" + args.dimm + "/vnc/" + args.temperature + "C/nist_results") +indir = os.path.normpath("data/" + args.dimm + "/vnc/" + args.temperature + "C/binaries") + +if not os.path.isfile(indir + "/lastrow.txt"): + input("I am going to overwrite 'lastrow.txt', proceed with caution") + init_file = open(indir + "/lastrow.txt", "w") + init_file.write(str(0) + "\n") + +# Hard-reset SoftMC before other experiments +reset_path = os.path.normpath("bin/SoftMC_reset") +reset = subprocess.Popen("exec sudo " + reset_path, shell=True, preexec_fn=os.setpgrp) +time.sleep(5) +os.system("sudo pkill -9 -P" + str(os.getpgid(reset.pid))) +reset.wait() + +# First run OBSERVE to collect von Neumann corrected bitstreams +observe_path = os.path.normpath("bin/OBSERVE " + indir) +observe_args = " > " + os.path.normpath(outdir + "/output.log") +observe = subprocess.Popen("exec sudo " + observe_path + observe_args, shell=True, preexec_fn=os.setpgrp) + +# Watch the bitstream directory for new files +before = ["lastrow.txt"] + +while True: + time.sleep(1) + after = os.listdir(indir) + s = set(before) + diff = [x for x in after if x not in s] + before = after + if len(diff) == 0: + continue + + bitstream = [[] for i in range(8)] # at most 8 STS runs + + # Partition + i = 0 + for fname in diff: + bitstream[i].append(fname) + i = (i+1) % 8 + + found = False + for i in range(len(bitstream[0])): + print("Finished " + str(i) + " out of " + str(len(bitstream[7]))) + processes = [] + for thr in range(0,8): + if len(bitstream[thr]) <= i: + break + command_output = os.path.normpath(outdir + "/thr" + str(thr)) + command_input = os.path.normpath(indir + '/' + bitstream[thr][i]) + command = 'tools/sts -O -i 1 -I 1 -S '+ str(1024*1024) +' -w ' + command_output +' -F r ' + command_input + " 2>/dev/null" + process = subprocess.Popen(command, shell=True) + processes.append(process) + + # Collect statuses + output = [p.wait() for p in processes] + + for thr in range(0,8): + if len(bitstream[thr]) <= i: + break + res_txt = open(os.path.normpath(outdir + '/thr' + str(thr) + '/finalAnalysisReport.txt'), "r") + lines = res_txt.readlines() + n_success = 0 + for j in range(7, len(lines)): + line = lines[j].strip() + if not ("1" in line or "0" in line): + break + split = line.split() + if "/" in split[11]: + n_success += int(split[11].split("/")[0]) + res_txt.close() + print(bitstream[thr][i] + " succeeded in " + str(n_success) + "tests + bits: " + str(1024*1024)) + if n_success < 187: + os.system("rm -f " + indir + "/" + bitstream[thr][i]) + if n_success > 186: + os.system("mkdir " + outdir + "/" + bitstream[thr][i]) + os.system("cp " + outdir + '/thr' + str(thr) + "/finalAnalysisReport.txt " + outdir + "/" + bitstream[thr][i]) + if n_success == 188: + os.system("sudo pkill -9 -P" + str(os.getpgid(observe.pid))) + observe.wait() # wait until the process actually terminates + found = True + if found: + break + +print("Finally found one cell that passes all NIST tests!") + +# Reset SoftMC because last process was killed prematurely +reset_path = os.path.normpath("bin/SoftMC_reset") +reset = subprocess.Popen("exec sudo " + observe_path, shell=True, preexec_fn=os.setpgrp) +time.sleep(5) +os.system("sudo pkill -9 -P" + str(os.getpgid(reset.pid))) +reset.wait() diff --git a/sources/apps/QUAC-TRNG/scripts/sha_bitstreams.py b/sources/apps/QUAC-TRNG/scripts/sha_bitstreams.py new file mode 100644 index 0000000..fd4dbfb --- /dev/null +++ b/sources/apps/QUAC-TRNG/scripts/sha_bitstreams.py @@ -0,0 +1,87 @@ +import os +import argparse +import subprocess +import time + +parser = argparse.ArgumentParser(description = "Collects sha bitstreams for a data pattern") + +parser.add_argument('dimm', help="DIMM label under test") +parser.add_argument('pattern', help="data pattern to use") +parser.add_argument('temperature', help="DIMM temperature in centigrades") + +args = parser.parse_args() + +bitstreams_available = False + +if not os.path.isdir("data/" + args.dimm + "/sha"): + os.mkdir("data/" + args.dimm + "/sha") + +if os.path.isdir("data/" + args.dimm + "/sha/" + args.temperature + "C"): + bitstreams_available = True + +if not os.path.isdir("data/" + args.dimm + "/sha/" + args.temperature + "C"): + os.mkdir("data/" + args.dimm + "/sha/" + args.temperature + "C") + +infile = os.path.normpath("data/" + args.dimm + "/ent/" + args.temperature + "C/top.txt") +outdir = os.path.normpath("data/" + args.dimm + "/sha/" + args.temperature + "C") + +if not os.path.isdir("data/" + args.dimm + "/sha"): + os.mkdir("data/" + args.dimm + "/sha_nist_results") + +if not os.path.isdir("data/" + args.dimm + "/sha/" + args.temperature + "C"): + os.mkdir("data/" + args.dimm + "/sha_nist_results/" + args.temperature + "C") + +sha_output = os.path.normpath("data/" + args.dimm + "/sha_nist_results/" + args.temperature + "C") + +if not bitstreams_available: + sha_path = "bin/SHA" + print("sudo " + sha_path + " " + outdir + " " + infile) + os.system("sudo " + sha_path + " " + outdir + " " + infile) +else: + print("I think we have the bitstreams available, I won't run softmc code again") + +i=0 +bitstream = [] + +for j in range(0,10): + bitstream.append([]) + +for fname in os.listdir(outdir): + bitstream[i].append(fname) + i+=1 + i = i % 10 + +print(bitstream) + +for i in range(len(bitstream[9])): + print("Finished " + str(i) + " out of " + str(len(bitstream[9]))) + processes = [] + for thr in range(0,10): + outpath = os.path.normpath(sha_output+'/thr'+str(thr)) + if not os.path.isdir(outpath): + os.mkdir(outpath) + command = 'tools/sts -O -i 1024 -I 1 -P 11=0.001 -S '+ str(1024*1024) +' -w '+ outpath +' -F r ' + outdir+'/'+bitstream[thr][i] + " 2>/dev/null" + process = subprocess.Popen(command, shell=True) + processes.append(process) + print(command) + + # Collect statuses + output = [p.wait() for p in processes] + + for thr in range(0,10): + outpath = os.path.normpath(sha_output+'/thr'+str(thr)) + res_txt = open(outpath + '/finalAnalysisReport.txt', "r") + lines = res_txt.readlines() + n_success = 0 + for j in range(7, len(lines)): + line = lines[j].strip() + if not ("1" in line or "0" in line): + break + split = line.split() + if "/" in split[11]: + n_success += int(split[11].split("/")[0]) + res_txt.close() + print(bitstream[thr][i] + " succeeded in " + str(n_success) + "tests + bits: " + str(1024*1024)) + if n_success > 186: + os.system("mkdir "+sha_output+"/" + bitstream[thr][i]) + os.system("cp "+sha_output+'/thr'+str(thr)+"/finalAnalysisReport.txt "+sha_output+"/" + bitstream[thr][i]) diff --git a/sources/apps/QUAC-TRNG/scripts/top_ents.py b/sources/apps/QUAC-TRNG/scripts/top_ents.py new file mode 100644 index 0000000..2a74569 --- /dev/null +++ b/sources/apps/QUAC-TRNG/scripts/top_ents.py @@ -0,0 +1,35 @@ +import os +import argparse +import pandas as pd +import numpy as np + +parser = argparse.ArgumentParser(description = "Postprocess entropy files and obtain top-10 segments with the highest entropy") +parser.add_argument('dimm', help="DIMM label") +parser.add_argument('pattern', help="4-bit data pattern [1000, 1001, 0111, etc]") +parser.add_argument('temp', help="temperature in centigrades") +args = parser.parse_args() + +dimm = args.dimm +pattern = args.pattern +temp = args.temp + +no_segments = 8192 +no_bitlines = 512 * 128 + +fn = "data/" + dimm + "/ent/" + temp + "C/1_1000_128_32768_" + pattern + "_floats.bin" +dirname = "data/" + dimm + "/ent/" + temp + "C/" + +arr = np.fromfile(fn, dtype=np.float32) +arr = arr.reshape(no_segments, no_bitlines) +arr = arr.sum(axis = 1) +indices = arr.argsort()[-10:][::-1] +print(indices) +print(arr[indices]) + +log = open(dirname + "top.txt", "w") +log.write("10\n") +for index in indices: + # bank is always 1 + # multiply segment # by 4 to get starting row addr + log.write("1 " + str(int(index) * 4) + " " + str(arr[index]) + " " + pattern + "\n") +log.close() diff --git a/sources/apps/QUAC-TRNG/src/observe.cpp b/sources/apps/QUAC-TRNG/src/observe.cpp new file mode 100644 index 0000000..275c516 --- /dev/null +++ b/sources/apps/QUAC-TRNG/src/observe.cpp @@ -0,0 +1,726 @@ +#include "instruction.h" +#include "prog.h" +#include "platform.h" +#include <fstream> +#include <iostream> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <cstring> +#include <list> +#include <algorithm> +#include <iterator> +#include <iomanip> + +#define NEWFILTER + +using namespace std; + +#define NO_BANKS 2 +#define NO_ROWS 32*1024 + +// Stride register ids are fixed and should not be changed +// CASR should always be reg 0 +#define CASR 0 +// BASR should always be reg 1 +#define BASR 1 +// RASR should always be reg 2 +#define RASR 2 + +#define PATTERN_REG 12 +#define INV_PATTERN_REG 13 +#define TEMP_PATTERN_REG 14 + +#define BAR 3 +#define CAR 4 +#define RARBASE 5 +#define RAR1 6 +#define RAR2 9 + +#define ONE_REG 10 +#define ZERO_REG 11 + +#define ITER_REG 7 +#define CTR_REG 8 + +#define INIT_LOOP 15 + +Program genRowCopy(int t1, int t2, int row1_reg, int row2_reg, int bank_reg) +{ + Program ret; + // t1 -> ACT to PRE, t2 -> PRE to 2nd ACT + int sz = 3 + t1 + t2; + sz = (4-(sz%4)) + sz; + Mininst buff[sz]; + for(int i = 0 ; i < sz ; i++) + buff[i] = SMC_NOP(); + + // Overwrite with the actual sequence + buff[0] = SMC_ACT(bank_reg, 0, row1_reg, 0); + buff[1+t1] = SMC_PRE(bank_reg, 0, 0); + buff[2+t1+t2] = SMC_ACT(bank_reg, 0, row2_reg, 0); + + for(int i = 0 ; i < sz ; i+=4) + ret.add_inst(buff[i], buff[i+1], buff[i+2], buff[i+3]); + + return ret; +} + +/** + * To send an ACT -> SLEEP t1 -> PRE -> SLEEP t2 -> ACT sequence + * This seq. will use BAR RAR1 and RAR2 registers + */ +Program genActPreActSequence(int t1, int t2, int row1_reg, int row2_reg, int bank_reg) +{ + Program ret; + // t1 -> ACT to PRE, t2 -> PRE to 2nd ACT + int sz = 3 + t1 + t2; + sz = (4-(sz%4)) + sz; + Mininst buff[sz]; + for(int i = 0 ; i < sz ; i++) + buff[i] = SMC_NOP(); + + // Overwrite with the actual sequence + buff[0] = SMC_ACT(bank_reg, 0, row1_reg, 0); + buff[1+t1] = SMC_PRE(bank_reg, 0, 0); + buff[2+t1+t2] = SMC_ACT(bank_reg, 0, row2_reg, 0); + + for(int i = 0 ; i < sz ; i+=4) + ret.add_inst(buff[i], buff[i+1], buff[i+2], buff[i+3]); + + return ret; +} + + +/** + * This function will generate a block of instructions that will + * write to a range of rows. It assumes the bank will be precharged. + */ + +Program genCopyRange(int row_reg, int no_rows, int bank_reg, bool stripes) +{ + Program ret; + ret.add_inst(SMC_LI(no_rows, INIT_LOOP)); + ret.add_inst(SMC_ADD(INIT_LOOP, row_reg, INIT_LOOP)); // write until this row + + ret.add_inst(SMC_LI(0, ONE_REG)); + ret.add_inst(SMC_LI(1, ZERO_REG)); + + ret.add_label("genWriteRange:LOOP_BEGIN"); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_below(genRowCopy(4,4,ONE_REG,row_reg,bank_reg)); + ret.add_inst(SMC_SLEEP(6)); + ret.add_inst(SMC_PRE(bank_reg, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_ADDI(row_reg, 1, row_reg)); + + ret.add_inst(SMC_SLEEP(6)); + ret.add_below(genRowCopy(4,4,ZERO_REG,row_reg,bank_reg)); + ret.add_inst(SMC_SLEEP(6)); + ret.add_inst(SMC_PRE(bank_reg, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_ADDI(row_reg, 1, row_reg)); + + ret.add_inst(SMC_SLEEP(6)); + + ret.add_below(genRowCopy(4,4,ONE_REG,row_reg,bank_reg)); + ret.add_inst(SMC_SLEEP(6)); + ret.add_inst(SMC_PRE(bank_reg, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_ADDI(row_reg, 1, row_reg)); + ret.add_inst(SMC_SLEEP(6)); + + ret.add_below(genRowCopy(4,4,ZERO_REG,row_reg,bank_reg)); + ret.add_inst(SMC_SLEEP(6)); + ret.add_inst(SMC_PRE(bank_reg, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_ADDI(row_reg, 1, row_reg)); + ret.add_inst(SMC_SLEEP(6)); + + ret.add_inst(SMC_SUBI(row_reg, no_rows, row_reg)); + return ret; +} + +/** + * This function will generate a block of instructions that will + * write to a range of rows. It assumes the bank will be precharged. + */ +Program genWriteRange1000(int row_reg, int no_rows, int bank_reg) +{ + Program ret; + ret.add_inst(SMC_LI(no_rows, INIT_LOOP)); + ret.add_inst(SMC_ADD(INIT_LOOP, row_reg, INIT_LOOP)); // write until this row + + ret.add_label("genWriteRange:LOOP_BEGIN"); + ret.add_inst(SMC_LI(0, CAR)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + + ret.add_inst(SMC_SUBI(row_reg, no_rows, row_reg)); + return ret; +} + +/** + * This function will generate a block of instructions that will + * write to a range of rows. It assumes the bank will be precharged. + */ + +Program genWriteRange(int row_reg, int no_rows, int bank_reg, bool stripes) +{ + Program ret; + ret.add_inst(SMC_LI(no_rows, INIT_LOOP)); + ret.add_inst(SMC_ADD(INIT_LOOP, row_reg, INIT_LOOP)); // write until this row + + ret.add_label("genWriteRange:LOOP_BEGIN"); + ret.add_inst(SMC_LI(0, CAR)); + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + for(int i = 0 ; i < 64 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + if(stripes) + { + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + } + ret.add_branch(ret.BR_TYPE::BL, row_reg, INIT_LOOP, "genWriteRange:LOOP_BEGIN"); + ret.add_inst(SMC_SUBI(row_reg, no_rows, row_reg)); + return ret; +} + +Program genReadRange(int row_reg, int no_rows, int bank_reg) +{ + Program ret; + ret.add_inst(SMC_LI(no_rows, INIT_LOOP)); + ret.add_inst(SMC_ADD(INIT_LOOP, row_reg, INIT_LOOP)); // read until this row + ret.add_label("genReadRange:LOOP_BEGIN"); + ret.add_inst(SMC_LI(0, CAR)); // TODO: fix this + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + for(int i = 0 ; i < 64 ; i++) + { + ret.add_inst(SMC_READ(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_READ(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_branch(ret.BR_TYPE::BL, row_reg, INIT_LOOP, "genReadRange:LOOP_BEGIN"); + /* + ret.add_inst(SMC_SLEEP(5)); + ret.dump_registers(); + ret.add_inst(SMC_END()); + */ + ret.add_inst(SMC_SUBI(row_reg, no_rows, row_reg)); + return ret; +} +/** + * @return an instruction formed by NOPs + */ +Inst all_nops() +{ + return __pack_mininsts(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()); +} + +Program gen_test_prog(int bank, int r1, int r2,int iters) +{ + Program program; + + program.add_inst(SMC_LI(8, CASR)); // Load 8 into CASR since each READ reads 8 columns + program.add_inst(SMC_LI(1, BASR)); // Load 1 into BASR + program.add_inst(SMC_LI(1, RASR)); // Load 1 into RASR + + program.add_inst(SMC_LI(0xffffffff, PATTERN_REG)); + program.add_inst(SMC_LI(0x00000000, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + program.add_inst(SMC_LDWD(PATTERN_REG,i)); + + program.add_inst(SMC_LI(bank, BAR)); + program.add_inst(SMC_LI(r1-(r1%4), RARBASE)); + + program.add_inst(SMC_LI(iters, ITER_REG)); + program.add_inst(SMC_LI(0, CTR_REG)); + + program.add_label("WRITE_BEGIN"); + program.add_inst(SMC_PRE(BAR, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + program.add_inst(SMC_SLEEP(3)); + program.add_below(genWriteRange(RARBASE, 4, BAR, true)); + //program.add_below(genWriteRange1000(RARBASE, 4, BAR)); + //program.add_below(genCopyRange(RARBASE, 4, BAR, true)); + program.add_inst(SMC_LI(r1, RAR1)); + program.add_inst(SMC_LI(r2, RAR2)); + + program.add_below(genActPreActSequence(1, 1, RAR1, RAR2, BAR)); + + program.add_inst(SMC_SLEEP(6)); + program.add_inst(SMC_PRE(BAR, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + + program.add_below(genReadRange(RARBASE, 1, BAR)); + program.add_inst(SMC_ADDI(CTR_REG, 1, CTR_REG)); + + program.add_branch(program.BR_TYPE::BL, CTR_REG, ITER_REG, "WRITE_BEGIN"); + program.add_inst(SMC_END()); + + return program; +} + +bool nonoverlapping_pattern(std::vector<uint8_t> bv) +{ + unsigned wsize = bv.size()/8; + uint8_t buf[wsize]; + for(int j = 0 ; j < wsize ; j++) + { + uint8_t intermediate = 0; + for(int k = 0 ; k < 8 ; k++) + intermediate = (intermediate << 1) | bv[j*8 + k]; + buf[j] = intermediate; + } + + int bin[8] = {0}; + for (int i = 0 ; i < (1024*128*8 - 3) ; i += 3) + { + if (i % 8 < 6) + bin[(buf[i/8] >> (i%8)) & 0x7]++; + else + { + int next_bits = 3 - (8-(i%8)); + int next_mask = next_bits == 2 ? 0x3 : 0x1; + int this_mask = next_bits == 2 ? 0x1 : 0x3; + uint8_t next_pattern = (buf[(i/8)+1]) & next_mask; + uint8_t this_pattern = (buf[i/8] >> (i%8)) & this_mask; + bin[(next_pattern << (3-next_bits)) | this_pattern]++; + } + } + + bool pass = true; + for (int i = 0 ; i < 8 ; i++) + { + cout << i << ": " << bin[i] << endl; + if (bin[i] < 30000 || bin[i] > 55000) + pass = false; + } + return pass; +} + +vector<int> pre_filter(SoftMCPlatform& platform, int bank, int row) +{ + vector<int> cells_to_track; + + const int NO_CELLS = 64 * 1024; + const int pre_pre_filter_iterations = 128*1024; + const int threshold_size = 4096; + + Program prog = gen_test_prog(bank, row, row+3, pre_pre_filter_iterations); + platform.execute(prog); + uint8_t buf1[8*1024]; + uint8_t buf2[8*1024]; + + vector<vector<uint8_t>> vnc_bitstreams; + + for (int i = 0 ; i < NO_CELLS ; i++) + vnc_bitstreams.push_back(vector<uint8_t>()); + + // Von neumann corrector first + for (int i = 0 ; i < pre_pre_filter_iterations/2 ; i++) + { + platform.receiveData((char*)buf1, 8*1024); + platform.receiveData((char*)buf2, 8*1024); + + for (int b = 0 ; b < NO_CELLS ; b++) + { + if(vnc_bitstreams[b].size() == threshold_size) + continue; + + int b1 = (buf1[b/8] >> b%8) & 0x1; + int b2 = (buf2[b/8] >> b%8) & 0x1; + + if (~b1 & b2) + vnc_bitstreams[b].push_back(1); + else if (b1 & ~b2) + vnc_bitstreams[b].push_back(0); + } + } + + int cnt = 0; + for (int b = 0 ; b < NO_CELLS ; b++) + { + if (vnc_bitstreams[b].size() == threshold_size) + cnt++; + } + + cout << cnt << " cells have passed initial filtering." << endl; + + // counter bins for each bitline + int bin[NO_CELLS][8]; + for (int i = 0 ; i < NO_CELLS ; i++) + { + for (int j = 0 ; j < 8 ; j++) + bin[i][j] = 0; + } + + int freq[NO_CELLS] = {0}; + + for (int b = 0 ; b < NO_CELLS ; b++) + { + if (vnc_bitstreams[b].size() < threshold_size - 1) + continue; + for (int i = 0 ; i < (threshold_size - 1)/3 ; i++) + { + int b1 = vnc_bitstreams[b][i] & 0x1; + int b2 = vnc_bitstreams[b][i+1] & 0x1; + int b3 = vnc_bitstreams[b][i+2] & 0x1; + + if (b1) + freq[b]++; + if (b2) + freq[b]++; + if (b3) + freq[b]++; + + bin[b][(b3 << 2) | (b2 << 1) | b1]++; + } + } + + bool passed_frequency[NO_CELLS] = {false}; + + int upper_freq = threshold_size/2 + threshold_size/2/100; + int lower_freq = threshold_size/2 - threshold_size/2/100; + for (int i = 0 ; i < NO_CELLS ; i++) + { + if (vnc_bitstreams[i].size() < threshold_size - 1) + continue; + if (freq[i] < upper_freq && freq[i] > lower_freq) + passed_frequency[i] = true; + } + + cnt = 0; + for (int b = 0 ; b < NO_CELLS ; b++) + { + if (passed_frequency[b]) + cnt++; + } + + cout << cnt << " cells have passed frequency filtering." << endl; + + //cout << "Cell: " << i << endl; + int lower_limit = ((threshold_size - 1) / 3 / 8) - (((threshold_size - 1) / 3 / 8) / 10); + int upper_limit = ((threshold_size - 1) / 3 / 8) + (((threshold_size - 1) / 3 / 8) / 10); + cout << "UL: " << upper_limit << " LL: " << lower_limit << endl; + bool passed[NO_CELLS]; + + for (int b = 0 ; b < NO_CELLS ; b++) + passed[b] = true; + + for (int i = 0 ; i < NO_CELLS ; i++) + { + if (!passed_frequency[i]) + { + passed[i] = false; + continue; + } + for (int j = 0 ; j < 8 ; j++) + { + //cout << "Bin" << j << ": " << bin[i][j] << endl; + if (bin[i][j] < lower_limit || bin[i][j] > upper_limit) + { + //cout << "Cell " << i << " Bin" << j << ": " << bin[i][j] << endl; + passed[i] = false; + break; + } + } + if (passed[i]) + cout << i << " " << "PASSed" << endl; + } + + for (int i = 0 ; i < NO_CELLS ; i++) + { + if (passed[i]) + cells_to_track.push_back(i); + } + + return cells_to_track; +} + +int main(int argc, char *argv[]) +{ + SoftMCPlatform platform; + int err; + + // buffer allocated for reading data from the board + uint8_t buf[8*1024]; + uint8_t prev_read[8*1024]; + + // Initialize the platform, opens file descriptors for the board PCI-E interface. + if((err = platform.init()) != SOFTMC_SUCCESS){ + cerr << "Could not initialize SoftMC Platform: " << err << endl; + } + // reset the board to hopefully restore the board's state + platform.reset_fpga(); + platform.set_aref(true); + + ifstream lfFile; + string trackFn = string(argv[1]) + "/" + "lastrow.txt"; + lfFile.open(trackFn); + + int x=0; + lfFile >> x; + + lfFile.close(); + + for(int bank = 1 ; bank < NO_BANKS ; bank++) + { + for(int r1 = x ; r1 < NO_ROWS ; r1 += 4) + { + bool satisfied = false; + #ifdef NEWFILTER + vector<int> cell_pos = pre_filter(platform, bank, r1); + #else + // Enrollment phase: + // Look at switching activity of each cell in a row + Program prog = gen_test_prog(bank, r1, r1+3, 32*1024); + platform.execute(prog); + int n_switches[64*1024]; // how many times each cell have switched + fill(n_switches, n_switches + 64*1024, 0); + platform.receiveData((char*)prev_read, 8*1024); + for(int k = 0; k < 32*1024-1 ; k++) + { + platform.receiveData((char*)buf, 8*1024); // read one segment each iteration + for(int i = 0 ; i < 8 * 1024 ; i++) + { + uint8_t diff = prev_read[i] ^ buf[i]; + for(int j = 0 ; j < 8 ; j++) + n_switches[i*8 + j] += (diff >> j) & 0x1; + } + copy(begin(buf), end(buf), begin(prev_read)); + } + vector<int> cell_pos; // Switching cell positions + float total_bps = 0; + for(int i = 0 ; i < 8*8*1024 ; i++) + { + // In total it takes 132ms to execute all the DRAM commands + // We compute bps based on switching + total_bps += (1000/132.f) * n_switches[i]; + if(n_switches[i] > 1000) + cell_pos.push_back(i); + } + + cout << "Finished enrollment for bank " << bank << + " row " << r1 << "... " << cell_pos.size() << " cells passed... Predicted bps " + << fixed << setw(11) << setprecision(6) << total_bps << endl; + #endif + + cout << "Finished enrollment for bank " << bank << + " row " << r1 << "... " << cell_pos.size() << " cells passed..." << endl; + + if(cell_pos.size() == 0) + continue; + + for (int i = 0 ; i < cell_pos.size() ; i++) + cout << cell_pos[i] << " "; + cout << endl; + + // Begin filtered acquisition phase + vector<vector<uint8_t>> stream_per_cell; + for(int i = 0 ; i < cell_pos.size() ; i++) + { + stream_per_cell.push_back(vector<uint8_t>()); + } + + int prev_bpc_i = 0; + int prev_cell_flips[cell_pos.size()]{0}; + + int loop_count = 0; + int loop_limit = 15; + bool bcf = false; // has the best cell exceeded the threshold + + while(!satisfied) + { + if(loop_count == loop_limit) + { + cout << "Exiting because we have iterated enough times" << endl; + break; + } + int iterations = 1024*1024; + Program prog = gen_test_prog(bank, r1, r1+3, iterations); + platform.execute(prog); + + for(int k = 0; k < iterations/2 ; k++) + { + platform.receiveData((char*)prev_read, 8*1024); // read one segment each iteration + platform.receiveData((char*)buf, 8*1024); // read one segment each iteration + //VON NEUMANN CORRECTOR BELOW + for(int i = 0 ; i < cell_pos.size() ; i++) + { + uint8_t b1 = (prev_read[cell_pos[i]/8] >> (cell_pos[i]%8)) & 0x1; + uint8_t b2 = (buf[cell_pos[i]/8] >> (cell_pos[i]%8)) & 0x1; + uint8_t b3 = b1 ^ b2; + if(!b3) continue; + if(b2) stream_per_cell[i].push_back(1); + else stream_per_cell[i].push_back(0); + } + } + + int bpc_flips = 0; // best performing cell flip count + int bpc_i = 0; // best performing cell index + for(int i = 0 ; i < cell_pos.size() ; i++) + { + if(stream_per_cell[i].size() > bpc_flips) + { + bpc_flips = stream_per_cell[i].size(); + bpc_i = i; + } + } + + /* + cout << "Dumping # of switches per cell..." << endl; + for(int i = 0 ; i < cell_pos.size() ; i++) + { + cout << "Cell#" << cell_pos[i] << " Got:" << stream_per_cell[i].size() - prev_cell_flips[i] + << "Exp:" << 32*n_switches[cell_pos[i]] << endl; + } + */ + + if(!bcf) + { + if(prev_bpc_i == bpc_i) + { + int remaining_bits = 1024*1024 - bpc_flips; + int count_diff = bpc_flips - prev_cell_flips[bpc_i]; + if(remaining_bits/count_diff + 1 > 16) + { + cout << "Exiting because it will take too long" << endl; + break; + } + if(remaining_bits < 0){ + cout << "Finished one cell, will keep iterating for 5 iterations" << endl; + bcf = true; + loop_limit = loop_count + 5; + } + else{ + cout << "It will take " << remaining_bits/count_diff + 1 << " more iterations to finish" << endl; + cout << "Based on the best cell's performance which has flipped " << bpc_flips << " times so far" << endl; + } + }else + { + cout << "Cannot approximate test time because the best performing cell changed" << endl; + cout << "Was " << cell_pos[prev_bpc_i] << " Now " << cell_pos[bpc_i] << endl; + } + } + cout << "Cells about to finish: " << endl; + for(int i = 0 ; i < cell_pos.size() ; i++){ + if(stream_per_cell[i].size() > 768*1024) + cout << cell_pos[i] << ":" << stream_per_cell[i].size() << " "; + + prev_cell_flips[i] = stream_per_cell[i].size(); + } + cout << endl; + prev_bpc_i = bpc_i; + loop_count ++; + } + + cout << "Writing bitstreams..." << endl; + int filtered_streams = 0; + for(int i = 0 ; i < cell_pos.size() ; i++){ + if(stream_per_cell[i].size() >= 1024*1024){ //&& nonoverlapping_pattern(stream_per_cell[i])){ + ofstream dump_file; + filtered_streams++; + string fn = string(argv[1]) + "/" + to_string(bank) + "_" + to_string(r1) + + "_" + to_string(r1+3) + "_" + to_string(cell_pos[i]); + dump_file.open(fn, ofstream::binary); + unsigned wsize = stream_per_cell[i].size()/8; + uint8_t wbuf[wsize]; + for(int j = 0 ; j < wsize ; j++) + { + uint8_t intermediate = 0; + for(int k = 0 ; k < 8 ; k++) + intermediate = (intermediate << 1) | stream_per_cell[i][j*8 + k]; + wbuf[j] = intermediate; + } + dump_file.write((char*)wbuf, wsize); + dump_file.close(); + } + } + //cout << "Filtered streams: " << filtered_streams << endl; + cout << "Finished processing b:" << bank << " r1:" << r1 << " r2:" << r1+3 << endl; + ofstream lfFile; + string trackFn = string(argv[1]) + "/" + "lastrow.txt"; + lfFile.open(trackFn); + lfFile << r1; + lfFile.close(); + } + } +} diff --git a/sources/apps/QUAC-TRNG/src/slow_entropy.cpp b/sources/apps/QUAC-TRNG/src/slow_entropy.cpp new file mode 100644 index 0000000..a18cf6e --- /dev/null +++ b/sources/apps/QUAC-TRNG/src/slow_entropy.cpp @@ -0,0 +1,204 @@ +#include "instruction.h" +#include "prog.h" +#include "platform.h" +#include "util.h" +#include <fstream> +#include <iostream> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <cstring> +#include <list> +#include <math.h> +#include <algorithm> +#include <iterator> +#include <iomanip> + +using namespace std; + +Program gen_test_prog(int bank, int r1, int r2, int iters, int read_cols, int placement) +{ + Program program; + + program.add_inst(SMC_LI(8, CASR)); // Load 8 into CASR since each READ reads 8 columns + program.add_inst(SMC_LI(1, RASR)); // Load 1 into RASR + + program.add_inst(SMC_LI(bank, BAR)); + program.add_inst(SMC_LI(r1, RARBASE)); + + program.add_inst(SMC_LI(iters, ITER_REG)); + program.add_inst(SMC_LI(0, CTR_REG)); + + program.add_inst(SMC_PRE(BAR,0,1),SMC_NOP(),SMC_NOP(),SMC_NOP()); + program.add_inst(SMC_SLEEP(5)); + + program.add_inst(SMC_LI(r1, RAR1)); + program.add_inst(SMC_LI(r2, RAR2)); + + program.add_label("WRITE_BEGIN"); + //program.add_below(genWriteRange(RARBASE, 4, BAR, true)); + switch(placement) + { + // Invert 0-1 start pattern before here when we set the + // WIDEREG + case 1000: + program.add_below(genWriteRange1000(RARBASE, 4, BAR)); + break; + case 1001: + program.add_below(genWriteRange1001(RARBASE, 4, BAR)); + break; + case 1010: + program.add_below(genWriteRange1010(RARBASE, 4, BAR)); + break; + case 1011: + program.add_below(genWriteRange1011(RARBASE, 4, BAR)); + break; + case 1100: + program.add_below(genWriteRange1100(RARBASE, 4, BAR)); + break; + case 1101: + program.add_below(genWriteRange1101(RARBASE, 4, BAR)); + break; + case 1110: + program.add_below(genWriteRange1110(RARBASE, 4, BAR)); + break; + case 1111: + program.add_below(genWriteRange1111(RARBASE, 4, BAR)); + break; + default: + printf("Unexpected placement encoding\n"); + exit(0); + } + + program.add_below(genActPreActSequence(1, 1, RAR1, RAR2, BAR)); + program.add_inst(SMC_SLEEP(4)); + program.add_inst(SMC_PRE(BAR, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + program.add_inst(SMC_SLEEP(4)); + program.add_below(genReadRange(RARBASE, 1, BAR, read_cols)); + + program.add_inst(SMC_ADDI(CTR_REG, 1, CTR_REG)); + program.add_branch(program.BR_TYPE::BL, CTR_REG, ITER_REG, "WRITE_BEGIN"); + program.add_inst(SMC_END()); + + return program; +} + +int main(int argc, char *argv[]) +{ + bool DUMP_RAW = false; + + SoftMCPlatform platform; + int err; + + // buffer allocated for reading data from the board + uint8_t buf[4*READ_CL*64]; + + // Initialize the platform, opens file descriptors for the board PCI-E interface. + if((err = platform.init()) != SOFTMC_SUCCESS){ + cerr << "Could not initialize SoftMC Platform: " << err << endl; + } + // reset the board to hopefully restore the board's state + platform.reset_fpga(); + platform.set_aref(true); + + int bank = 1; + + int row_stride = atoi(argv[3]); + + string placement = argv[2]; + bool invert = false; + + int iters = atoi(argv[4]); + + ofstream ent_file; + string fn = string(argv[1]) + "/" + to_string(bank) + "_" + to_string(iters) + "_" + to_string(READ_CL) + "_" + to_string(NO_ROWS) + "_" + placement + ".txt"; + cout << "Saving to file " << fn << endl; + ent_file.open(fn); + + char new_placement[10]; + if (placement[0] == '0') + { + for (std::string::size_type i = 0 ; i < placement.size() ; i++) + if(placement[i] == '0') + new_placement[i] = '1'; + else if(placement[i] == '1') + new_placement[i] = '0'; + else + new_placement[i] = placement[i]; + placement = string(new_placement); + invert = true; + } + + Program program; + program.add_inst(SMC_LI(invert ? 0x0 : 0xffffffff, PATTERN_REG)); + program.add_inst(SMC_LI(invert ? 0xffffffff : 0x0, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + program.add_inst(SMC_LDWD(PATTERN_REG,i)); + program.add_inst(SMC_END()); + platform.execute(program); + + unsigned char *byte_buf[READ_CL*64]; + for(int i = 0 ; i < READ_CL*64 ; i++) + byte_buf[i] = new unsigned char[iters]; + + // Frequency of 1s and 0s generated by each bitline + int alphabet[READ_CL*64*8][2]; + + ent_file << NO_ROWS/row_stride << " " << READ_CL << endl; + + for(int r1 = 0 ; r1 < NO_ROWS ; r1+=row_stride) + { + ent_file << r1 << " "; + Program prog = gen_test_prog(bank, r1, r1+3, iters, READ_CL, std::stoi(placement)); + platform.execute(prog); + + for(int i = 0 ; i < iters ; i++) + { + platform.receiveData((char*)buf, READ_CL*64); // read one segment each iteration + for(int j = 0 ; j < READ_CL*64 ; j++) + byte_buf[j][i] = buf[j]; + } + + for(int i = 0 ; i < READ_CL*64*8 ; i++) + for(int j = 0 ; j < 2 ; j++) + alphabet[i][j] = 0; + + for(int j = 0 ; j < READ_CL*64*8 ; j++) + { + for(int i = 0 ; i < iters ; i++) + alphabet[j][(byte_buf[j/8][i]>>(j%8))&0x1] += 1; + + float approxent = 0; + for(int i = 0 ; i < 2 ; i++) + { + int freq = alphabet[j][i]; + float p = ((float) freq) / ((float) iters); + approxent -= freq == 0 ? 0 : p * log2(p); + } + + ent_file << approxent << " "; + + /* + if(approxent > 0.995) + { + unsigned char bits[iters/8]{0}; + for(int i = 0 ; i < iters ; i++) + bits[i/8] |= ((byte_buf[j/8][i] >> (j%8)) & 0x1) << (7-(i%8)); + ofstream dump_file; + string fn = string(argv[2]) + "/" + to_string(r1) + "_" + to_string(j) + ".bin"; + dump_file.open(fn, ofstream::binary); + dump_file.write((char*)(bits), iters/8); + dump_file.close(); + } + */ + } + + ent_file << endl; + + printf("\rrow %d", r1); + fflush(stdout); + } + for(int i = 0 ; i < READ_CL*64 ; i++) + delete byte_buf[i]; + ent_file.close(); +} diff --git a/sources/apps/QUAC-TRNG/src/slow_sha.cpp b/sources/apps/QUAC-TRNG/src/slow_sha.cpp new file mode 100644 index 0000000..69bf72f --- /dev/null +++ b/sources/apps/QUAC-TRNG/src/slow_sha.cpp @@ -0,0 +1,233 @@ +#include "instruction.h" +#include "prog.h" +#include "platform.h" +#include <fstream> +#include <iostream> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <cstring> +#include <list> +#include <algorithm> +#include <iterator> +#include <iomanip> +#include <openssl/sha.h> +#include "util.h" + +using namespace std; + +// each iter generates 320 bytes of RN +// #define ITERS_SHA 450000 +#define READ_CL 128 + +// Stride register ids are fixed and should not be changed +// CASR should always be reg 0 +#define CASR 0 +// BASR should always be reg 1 +#define BASR 1 +// RASR should always be reg 2 +#define RASR 2 + +#define PATTERN_REG 12 +#define INV_PATTERN_REG 13 +#define TEMP_PATTERN_REG 14 + +#define BAR 3 +#define CAR 4 +#define RARBASE 5 +#define RAR1 6 +#define RAR2 9 + +#define COPY_REG1 10 +#define COPY_REG2 11 + +#define ITER_REG 7 +#define CTR_REG 8 + +#define INIT_LOOP 15 + +Program gen_test_prog(int bank, int r1, int r2, int iters, int placement) +{ + Program program; + + program.add_inst(SMC_LI(8, CASR)); // Load 8 into CASR since each READ reads 8 columns + program.add_inst(SMC_LI(1, BASR)); // Load 1 into BASR + program.add_inst(SMC_LI(1, RASR)); // Load 1 into RASR + + program.add_inst(SMC_LI(bank, BAR)); + program.add_inst(SMC_LI(r1-(r1%4), RARBASE)); + + program.add_inst(SMC_LI(iters, ITER_REG)); + program.add_inst(SMC_LI(0, CTR_REG)); + + program.add_label("WRITE_BEGIN"); + program.add_inst(SMC_PRE(BAR, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + program.add_inst(SMC_SLEEP(3)); + switch(placement) + { + // Invert 0-1 start pattern before here when we set the + // WIDEREG + case 1000: + program.add_below(genWriteRange1000(RARBASE, 4, BAR)); + break; + case 1001: + program.add_below(genWriteRange1001(RARBASE, 4, BAR)); + break; + case 1010: + program.add_below(genWriteRange1010(RARBASE, 4, BAR)); + break; + case 1011: + program.add_below(genWriteRange1011(RARBASE, 4, BAR)); + break; + case 1100: + program.add_below(genWriteRange1100(RARBASE, 4, BAR)); + break; + case 1101: + program.add_below(genWriteRange1101(RARBASE, 4, BAR)); + break; + case 1110: + program.add_below(genWriteRange1110(RARBASE, 4, BAR)); + break; + case 1111: + program.add_below(genWriteRange1111(RARBASE, 4, BAR)); + break; + default: + printf("Unexpected placement encoding\n"); + exit(0); + } + + //program.add_below(genCopyRange(RARBASE, 4, BAR, true)); + program.add_inst(SMC_LI(r1, RAR1)); + program.add_inst(SMC_LI(r2, RAR2)); + + program.add_below(genActPreActSequence(1, 1, RAR1, RAR2, BAR)); + + program.add_inst(SMC_SLEEP(6)); + program.add_inst(SMC_PRE(BAR, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + + program.add_below(genReadRange(RARBASE, 1, BAR, 128)); + program.add_inst(SMC_ADDI(CTR_REG, 1, CTR_REG)); + + program.add_branch(program.BR_TYPE::BL, CTR_REG, ITER_REG, "WRITE_BEGIN"); + program.add_inst(SMC_END()); + + return program; +} + +int main(int argc, char *argv[]) +{ + SoftMCPlatform platform; + int err; + + // buffer allocated for reading data from the board + unsigned char buf[4*READ_CL*64]; + + // Initialize the platform, opens file descriptors for the board PCI-E interface. + if((err = platform.init()) != SOFTMC_SUCCESS){ + cerr << "Could not initialize SoftMC Platform: " << err << endl; + } + // reset the board to hopefully restore the board's state + platform.reset_fpga(); + platform.set_aref(true); + // Read selected segments from file + // first line: <number of segments> + // each line: <bank> <segment_start> <placement> + ifstream sf; + string sf_name = string(argv[2]); + sf.open(sf_name); + + int n_segments; sf >> n_segments; + std::vector<std::string> placements; + std::vector<int> banks, segments; + std::vector<float> ents; + for(int i = 0 ; i < n_segments ; i++) + { + int read; + sf >> read; banks.push_back(read); + sf >> read; segments.push_back(read); + float ent; + sf >> ent; ents.push_back(ent); + std::string rd; + sf >> rd; placements.push_back(rd); + } + + for(int i = 0 ; i < n_segments ; i++) + { + int bank = banks[i]; + int r1 = segments[i]; + float ent = ents[i]; + + if (ent < 1) + continue; + + std::string placement = placements[i]; + + bool invert = false; + // placement_magic + char new_placement[10]; + if (placement[0] == '0') + { + for (std::string::size_type i = 0 ; i < placement.size() ; i++) + if(placement[i] == '0') + new_placement[i] = '1'; + else if(placement[i] == '1') + new_placement[i] = '0'; + else + new_placement[i] = placement[i]; + placement = string(new_placement); + invert = true; + } + + Program program; + program.add_inst(SMC_LI(invert ? 0x0 : 0xffffffff, PATTERN_REG)); + program.add_inst(SMC_LI(invert ? 0xffffffff : 0x0, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + program.add_inst(SMC_LDWD(PATTERN_REG,i)); + program.add_inst(SMC_END()); + platform.execute(program); + + unsigned char *bitstream; + + unsigned int sha_input_ent = 256; + unsigned int sha_input_blocks = ent/sha_input_ent + 1; + unsigned int row_partition_size = 8192/sha_input_blocks; + // UNTODO //TODO: from now on this is going to be 16Mbits + unsigned int test_iters = (1024*1024*1024)/(8*sha_input_blocks*32); + + + bitstream = new unsigned char[test_iters*32*sha_input_blocks]; + + cout << "Running for " << test_iters << " iterations." << endl; + cout << "Total bits of entropy: " << ent << " -> " << sha_input_blocks << " sha input blocks for the row" << endl; + + Program prog = gen_test_prog(bank, r1, r1+3, test_iters, std::stoi(placement)); + platform.execute(prog); + + for(int i = 0 ; i < test_iters ; i++) + { + platform.receiveData((char*)buf, 8192); // read one segment each iteration + // https://stackoverflow.com/q/13784434 + for(int j = 0 ; j < sha_input_blocks ; j++) + { + unsigned char hash[SHA256_DIGEST_LENGTH]; + SHA256_CTX sha256; + SHA256_Init(&sha256); + SHA256_Update(&sha256, (char*)(buf+row_partition_size*j), row_partition_size); + SHA256_Final(hash, &sha256); + for(int k = 0 ; k < SHA256_DIGEST_LENGTH ; k++) + { + bitstream[i*sha_input_blocks*SHA256_DIGEST_LENGTH + j*SHA256_DIGEST_LENGTH + k] = hash[k]; + } + } + } + ofstream dump_file; + string fn = string(argv[1]) + "/" + placements[i] + "_" + to_string(r1) + ".bin"; + dump_file.open(fn, ofstream::binary); + dump_file.write((char*)(bitstream), test_iters*32*sha_input_blocks); + dump_file.close(); + printf("\rrow %d", r1); + fflush(stdout); + delete bitstream; + } + +} diff --git a/sources/apps/QUAC-TRNG/src/util.cpp b/sources/apps/QUAC-TRNG/src/util.cpp new file mode 100644 index 0000000..41e4d50 --- /dev/null +++ b/sources/apps/QUAC-TRNG/src/util.cpp @@ -0,0 +1,670 @@ +#include "util.h" + +/** + * To send an ACT -> SLEEP t1 -> PRE -> SLEEP t2 -> ACT sequence + * This seq. will use BAR RAR1 and RAR2 registers + */ +Program genActPreActSequence(int t1, int t2, int row1_reg, int row2_reg, int bank_reg) +{ + Program ret; + // t1 -> ACT to PRE, t2 -> PRE to 2nd ACT + int sz = 3 + t1 + t2; + sz = (4-(sz%4)) + sz; + Mininst buff[sz]; + for(int i = 0 ; i < sz ; i++) + buff[i] = SMC_NOP(); + + // Overwrite with the actual sequence + buff[0] = SMC_ACT(bank_reg, 0, row1_reg, 0); + buff[1+t1] = SMC_PRE(bank_reg, 0, 0); + buff[2+t1+t2] = SMC_ACT(bank_reg, 0, row2_reg, 0); + + for(int i = 0 ; i < sz ; i+=4) + ret.add_inst(buff[i], buff[i+1], buff[i+2], buff[i+3]); + + return ret; +} + +Program genReadRange(int row_reg, int no_rows, int bank_reg, int read_cols) +{ + Program ret; + ret.add_inst(SMC_LI(no_rows, INIT_LOOP)); + ret.add_inst(SMC_ADD(INIT_LOOP, row_reg, INIT_LOOP)); // read until this row + ret.add_label("genReadRange:LOOP_BEGIN"); + ret.add_inst(SMC_LI(0, CAR)); + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < read_cols/4 ; i++) + { + ret.add_inst(SMC_READ(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_READ(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_READ(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_READ(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_branch(ret.BR_TYPE::BL, row_reg, INIT_LOOP, "genReadRange:LOOP_BEGIN"); + ret.add_inst(SMC_SUBI(row_reg, no_rows, row_reg)); + return ret; +} + + + + +/** + * This function will generate a block of instructions that will + * write to a range of rows. It assumes the bank will be precharged. + */ +Program genWriteRange1000(int row_reg, int no_rows, int bank_reg) +{ + Program ret; + ret.add_inst(SMC_LI(no_rows, INIT_LOOP)); + ret.add_inst(SMC_ADD(INIT_LOOP, row_reg, INIT_LOOP)); // write until this row + + ret.add_label("genWriteRange:LOOP_BEGIN"); + ret.add_inst(SMC_LI(0, CAR)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + + ret.add_inst(SMC_SUBI(row_reg, no_rows, row_reg)); + return ret; +} + +/** + * This function will generate a block of instructions that will + * write to a range of rows. It assumes the bank will be precharged. + */ +Program genWriteRange1001(int row_reg, int no_rows, int bank_reg) +{ + Program ret; + ret.add_inst(SMC_LI(no_rows, INIT_LOOP)); + ret.add_inst(SMC_ADD(INIT_LOOP, row_reg, INIT_LOOP)); // write until this row + + ret.add_label("genWriteRange:LOOP_BEGIN"); + ret.add_inst(SMC_LI(0, CAR)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_SUBI(row_reg, no_rows, row_reg)); + return ret; +} + +/** + * This function will generate a block of instructions that will + * write to a range of rows. It assumes the bank will be precharged. + */ +Program genWriteRange1010(int row_reg, int no_rows, int bank_reg) +{ + Program ret; + ret.add_inst(SMC_LI(no_rows, INIT_LOOP)); + ret.add_inst(SMC_ADD(INIT_LOOP, row_reg, INIT_LOOP)); // write until this row + + ret.add_label("genWriteRange:LOOP_BEGIN"); + ret.add_inst(SMC_LI(0, CAR)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + + ret.add_branch(ret.BR_TYPE::BL, row_reg, INIT_LOOP, "genWriteRange:LOOP_BEGIN"); + ret.add_inst(SMC_SUBI(row_reg, no_rows, row_reg)); + return ret; +} + +/** + * This function will generate a block of instructions that will + * write to a range of rows. It assumes the bank will be precharged. + */ +Program genWriteRange1011(int row_reg, int no_rows, int bank_reg) +{ + Program ret; + ret.add_inst(SMC_LI(no_rows, INIT_LOOP)); + ret.add_inst(SMC_ADD(INIT_LOOP, row_reg, INIT_LOOP)); // write until this row + + ret.add_label("genWriteRange:LOOP_BEGIN"); + ret.add_inst(SMC_LI(0, CAR)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_SUBI(row_reg, no_rows, row_reg)); + return ret; +} + +/** + * This function will generate a block of instructions that will + * write to a range of rows. It assumes the bank will be precharged. + */ +Program genWriteRange1100(int row_reg, int no_rows, int bank_reg) +{ + Program ret; + ret.add_inst(SMC_LI(no_rows, INIT_LOOP)); + ret.add_inst(SMC_ADD(INIT_LOOP, row_reg, INIT_LOOP)); // write until this row + + ret.add_label("genWriteRange:LOOP_BEGIN"); + ret.add_inst(SMC_LI(0, CAR)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + + ret.add_inst(SMC_SLEEP(4)); + + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + + ret.add_inst(SMC_SUBI(row_reg, no_rows, row_reg)); + return ret; +} + +/** + * This function will generate a block of instructions that will + * write to a range of rows. It assumes the bank will be precharged. + */ +Program genWriteRange1101(int row_reg, int no_rows, int bank_reg) +{ + Program ret; + ret.add_inst(SMC_LI(no_rows, INIT_LOOP)); + ret.add_inst(SMC_ADD(INIT_LOOP, row_reg, INIT_LOOP)); // write until this row + + ret.add_label("genWriteRange:LOOP_BEGIN"); + ret.add_inst(SMC_LI(0, CAR)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + + ret.add_inst(SMC_SLEEP(4)); + + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + + + ret.add_inst(SMC_SUBI(row_reg, no_rows, row_reg)); + return ret; +} + +/** + * This function will generate a block of instructions that will + * write to a range of rows. It assumes the bank will be precharged. + */ +Program genWriteRange1110(int row_reg, int no_rows, int bank_reg) +{ + Program ret; + ret.add_inst(SMC_LI(no_rows, INIT_LOOP)); + ret.add_inst(SMC_ADD(INIT_LOOP, row_reg, INIT_LOOP)); // write until this row + + ret.add_label("genWriteRange:LOOP_BEGIN"); + ret.add_inst(SMC_LI(0, CAR)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + ret.add_inst(SMC_MV(PATTERN_REG, TEMP_PATTERN_REG)); + ret.add_inst(SMC_MV(INV_PATTERN_REG, PATTERN_REG)); + ret.add_inst(SMC_MV(TEMP_PATTERN_REG, INV_PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + ret.add_inst(SMC_LDWD(PATTERN_REG,i)); + + ret.add_inst(SMC_SUBI(row_reg, no_rows, row_reg)); + return ret; +} + +/** + * This function will generate a block of instructions that will + * write to a range of rows. It assumes the bank will be precharged. + */ +Program genWriteRange1111(int row_reg, int no_rows, int bank_reg) +{ + Program ret; + ret.add_inst(SMC_LI(no_rows, INIT_LOOP)); + ret.add_inst(SMC_ADD(INIT_LOOP, row_reg, INIT_LOOP)); // write until this row + + ret.add_label("genWriteRange:LOOP_BEGIN"); + ret.add_inst(SMC_LI(0, CAR)); + + ret.add_inst(SMC_ACT(bank_reg, 0, row_reg, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_SLEEP(4)); + + for(int i = 0 ; i < 32 ; i++) + { + ret.add_inst(SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP(), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0), SMC_NOP()); + ret.add_inst(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_WRITE(bank_reg, 0, CAR, 1, 0, 0)); + ret.add_inst(SMC_NOP(),SMC_NOP(),SMC_NOP(),SMC_NOP()); + } + ret.add_inst(SMC_SLEEP(4)); + ret.add_inst(SMC_PRE(bank_reg, 0, 1), SMC_NOP(), SMC_NOP(), SMC_NOP()); + + ret.add_branch(ret.BR_TYPE::BL, row_reg, INIT_LOOP, "genWriteRange:LOOP_BEGIN"); + ret.add_inst(SMC_SUBI(row_reg, no_rows, row_reg)); + return ret; +} + +/** + * @return an instruction formed by NOPs + */ +Inst all_nops() +{ + return __pack_mininsts(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()); +} diff --git a/sources/apps/QUAC-TRNG/src/util.h b/sources/apps/QUAC-TRNG/src/util.h new file mode 100644 index 0000000..1f07e0e --- /dev/null +++ b/sources/apps/QUAC-TRNG/src/util.h @@ -0,0 +1,55 @@ +#ifndef UTIL_HH +#define UTIL_HH + +#include "prog.h" + +#define ITERS_DUMP 1024*1024 +#define READ_CL_DUMP 8 + +#define ITERS 10000 //1024*1024 // 256*8 +#define READ_CL 128 + +#define NO_BANKS 1 +#define NO_ROWS 1024*32 + +// Stride register ids are fixed and should not be changed +// CASR should always be reg 0 +#define CASR 0 +// BASR should always be reg 1 +#define BASR 1 +// RASR should always be reg 2 +#define RASR 2 + +#define PATTERN_REG 12 +#define INV_PATTERN_REG 13 +#define TEMP_PATTERN_REG 14 + +#define BAR 3 +#define CAR 4 +#define RARBASE 5 +#define RAR1 6 +#define RAR2 9 + +#define COPY_REG1 10 +#define COPY_REG2 11 + +#define ITER_REG 7 +#define CTR_REG 8 + +#define INIT_LOOP 15 + +Program genActPreActSequence(int t1, int t2, int row1_reg, int row2_reg, int bank_reg); +Program genReadRange(int row_reg, int no_rows, int bank_reg, int read_cols); +Program genWriteRange1000(int row_reg, int no_rows, int bank_reg); +Program genWriteRange1111(int row_reg, int no_rows, int bank_reg); +Program genWriteRange1001(int row_reg, int no_rows, int bank_reg); +Program genWriteRange1010(int row_reg, int no_rows, int bank_reg); +Program genWriteRange1011(int row_reg, int no_rows, int bank_reg); +Program genWriteRange1100(int row_reg, int no_rows, int bank_reg); +Program genWriteRange1101(int row_reg, int no_rows, int bank_reg); +Program genWriteRange1110(int row_reg, int no_rows, int bank_reg); +Inst all_nops(); + + + +#endif diff --git a/sources/apps/ResetBoard/SoftMC_reset b/sources/apps/ResetBoard/SoftMC_reset Binary files differnew file mode 100755 index 0000000..709b93c --- /dev/null +++ b/sources/apps/ResetBoard/SoftMC_reset diff --git a/sources/apps/ResetBoard/full_reset.sh b/sources/apps/ResetBoard/full_reset.sh new file mode 100755 index 0000000..577a9a9 --- /dev/null +++ b/sources/apps/ResetBoard/full_reset.sh @@ -0,0 +1,9 @@ +#!/bin/bash + + +($( dirname "${BASH_SOURCE[0]}" )/SoftMC_reset) & +pid=$! + +sleep 2 + +kill -9 $pid diff --git a/sources/apps/RetentionTest/Makefile b/sources/apps/RetentionTest/Makefile new file mode 100755 index 0000000..c3828af --- /dev/null +++ b/sources/apps/RetentionTest/Makefile @@ -0,0 +1,28 @@ +program_NAME := SoftMC_Retention +program_CXX_SRCS := retention.cpp $(wildcard ../../api/*.c) $(wildcard ../../api/*.cpp) +program_CXX_OBJS := ${program_CXX_SRCS:.cpp=.o} +program_CXX_OBJS := ${program_CXX_OBJS:.c=.o} +program_OBJS := $(program_CXX_OBJS) +program_INCLUDE_DIRS := ../../api ../../../boost-lib +program_LIBRARY_DIRS := +program_LIBRARIES := pthread +CPPFLAGS += -g -std=c++11 + +CPPFLAGS += $(foreach includedir,$(program_INCLUDE_DIRS),-I$(includedir)) +LDFLAGS += $(foreach librarydir,$(program_LIBRARY_DIRS),-L$(librarydir)) +LDFLAGS += $(foreach library,$(program_LIBRARIES),-l$(library)) + +CC=g++ + +.PHONY: all clean distclean + +all: $(program_NAME) + +$(program_NAME): $(program_OBJS) + $(CC) $(CPPFLAGS) $(program_OBJS) -o $(program_NAME) $(LDFLAGS) + +clean: + @- $(RM) $(program_NAME) + @- $(RM) $(program_OBJS) + +distclean: clean diff --git a/sources/apps/RetentionTest/retention.cpp b/sources/apps/RetentionTest/retention.cpp new file mode 100644 index 0000000..86b8d98 --- /dev/null +++ b/sources/apps/RetentionTest/retention.cpp @@ -0,0 +1,205 @@ +#include "instruction.h" +#include "prog.h" +#include "platform.h" +#include <fstream> +#include <iostream> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <cstring> +#include <list> +#include <cassert> +#include <sys/time.h> + +using namespace std; + +#define RETENTION_IN_MS 1000 +#define NUM_BANKS 1 +#define NUM_ROWS 32*1024 + +#define GET_TIME_INIT(num) struct timeval _timers[num] +#define GET_TIME_VAL(num) gettimeofday(&_timers[num], NULL) +#define TIME_VAL_TO_MS(num) (((double)_timers[num].tv_sec*1000.0) + ((double)_timers[num].tv_usec/1000.0)) + +// Need to hand-calculate below value? +// indicates how many rows we can write to +// before testing for retention +// TODO SET THIS TO A REASONABLE VALUE +#define MAX_ROW_PER_ITER 512 + +Inst all_nops() +{ + return __pack_mininsts(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()); +} + +int main(int argc, char*argv[]) +{ + Program program; + SoftMCPlatform platform; + int err; + + if(argc != 3) + { + printf("Usage: \n ./SoftMC_RETENTION <retention time in ms> <output file name>\n"); + exit(0); + } + + if((err = platform.init()) != SOFTMC_SUCCESS){ + cerr << "Could not initialize SoftMC Platform: " << err << endl; + } + + char* buf = (char*) malloc(sizeof(char)*32*1024); + platform.reset_fpga(); + // sleep(1); + + uint64_t sleep_counter = (uint64_t) ((double) RETENTION_IN_MS * 1000 * 1000 // retention in ns + / 6) ; // 6ns clock period + + assert(sleep_counter < (uint64_t)4*1024*1024*1024 && "Retention time too large to fit into a 32 bit register"); + + printf("Retention time given as %d ms\n", RETENTION_IN_MS); + printf("Will sleep for %ld cycles\n", sleep_counter*6); + + program.add_inst(SMC_LI(8, 0)); // CASR + program.add_inst(SMC_LI(1, 1)); // BASR + program.add_inst(SMC_LI(1, 2)); // RASR + + program.add_inst(SMC_LI(NUM_ROWS, 8)); // NUM_ROWS + program.add_inst(SMC_LI(NUM_BANKS, 11)); + + // To fill wide-reg with data + program.add_inst(SMC_LI(0xffffffff, 7)); + for(int i = 0 ; i < 16 ; i++) + program.add_inst(SMC_LDWD(7,i)); + + + program.add_inst(SMC_LI(0, 5)); // BAR + program.add_label("BANK_BEGIN"); + program.add_inst(SMC_LI(MAX_ROW_PER_ITER,7)); + program.add_inst(SMC_LI(0, 4)); // CAR + program.add_inst(SMC_LI(0, 6)); // RAR + + program.add_label("WRITE"); + + // PRE & wait for tRP + program.add_inst(__pack_mininsts( + SMC_PRE(5, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + )); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + + // ACT & wait for tRCD + program.add_inst(__pack_mininsts( + SMC_ACT(5, 0, 6, 1), + SMC_NOP(), SMC_NOP(), SMC_NOP() + )); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + + // Write to a whole row + for(int i = 0 ; i < 128 ; i++){ + program.add_inst(__pack_mininsts( + SMC_WRITE(5, 0, 4, 1, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + )); + program.add_inst(all_nops()); + } + + // Wait for t(write-precharge) + // & precharge the open bank + program.add_inst(all_nops()); + program.add_inst(all_nops()); + program.add_inst(__pack_mininsts( + SMC_PRE(5, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + )); + + program.add_branch(program.BR_TYPE::BL, 6, 7, "WRITE"); + + // implement "sleep" as a counter + //program.add_label("SLEEP"); + //program.add_inst(SMC_LI(0, 9)); + //program.add_inst(SMC_LI(sleep_counter/7, 10)); + //program.add_label("SLEEP_INNER"); + //program.add_inst(SMC_ADDI(9,1,9)); + //program.add_branch(program.BR_TYPE::BL, 9, 10, "SLEEP_INNER"); + program.add_inst(SMC_SLEEP(sleep_counter)); + + // decrement RAR to read from newly written rows + program.add_inst(SMC_SUBI(6, MAX_ROW_PER_ITER, 6)); + + program.add_label("READ"); + // PRE & wait for tRP + program.add_inst(__pack_mininsts( + SMC_PRE(5, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + )); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + + // ACT & wait for tRCD + program.add_inst(__pack_mininsts( + SMC_ACT(5, 0, 6, 1), + SMC_NOP(), SMC_NOP(), SMC_NOP() + )); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + + // Read from a whole row + for(int i = 0 ; i < 128 ; i++){ + program.add_inst(__pack_mininsts( + SMC_READ(5, 0, 4, 1, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + )); + program.add_inst(all_nops()); + } + + // Wait for t(read-precharge) + // & precharge the open bank + program.add_inst(all_nops()); + program.add_inst(all_nops()); + program.add_inst(__pack_mininsts( + SMC_PRE(5, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + )); + + program.add_branch(program.BR_TYPE::BEQ, 6, 8, "INC_BANK"); + program.add_branch(program.BR_TYPE::BL, 6, 7, "READ"); + program.add_inst(SMC_ADDI(7, MAX_ROW_PER_ITER, 7)); + program.add_branch(program.BR_TYPE::JUMP, 0, 0, "WRITE"); + + program.add_label("INC_BANK"); + program.add_inst(SMC_ADDI(5,1,5)); + program.add_branch(program.BR_TYPE::BEQ, 5, 11, "END"); + program.add_branch(program.BR_TYPE::JUMP,0,0,"BANK_BEGIN"); + + program.add_label("END"); + program.add_inst(SMC_END()); + + platform.execute(program); + + uint8_t pattern = 0xff; + uint64_t r_count = 0; + + ofstream failures_file; + failures_file.open(std::string(argv[2]), ofstream::binary); + while(1) + { + platform.receiveData(buf,8*1024); + for(int j = 0 ; j < 8*1024 ; j++) + failures_file << (buf[j] ^ pattern); + r_count++; + if(r_count % (NUM_ROWS) == 0) + printf("Bank %ld complete\n", r_count/(NUM_ROWS)); + if(r_count == NUM_BANKS*NUM_ROWS) + break; + printf("\rRead: %ld",r_count); + fflush(stdout); + } + printf("Test complete!\n"); +} diff --git a/sources/apps/SanityCheck/.gitignore b/sources/apps/SanityCheck/.gitignore new file mode 100644 index 0000000..5f2829a --- /dev/null +++ b/sources/apps/SanityCheck/.gitignore @@ -0,0 +1 @@ +SoftMC_Sanity
\ No newline at end of file diff --git a/sources/apps/SanityCheck/Makefile b/sources/apps/SanityCheck/Makefile new file mode 100755 index 0000000..19e8a5c --- /dev/null +++ b/sources/apps/SanityCheck/Makefile @@ -0,0 +1,33 @@ +program_NAME := SoftMC_Sanity +program_CXX_SRCS := sanity.cpp $(wildcard ../../api/*.c) $(wildcard ../../api/*.cpp) +program_CXX_OBJS := ${program_CXX_SRCS:.cpp=.o} +program_CXX_OBJS := ${program_CXX_OBJS:.c=.o} +program_OBJS := $(program_CXX_OBJS) +program_INCLUDE_DIRS := ../../api ../../../boost-lib +program_LIBRARY_DIRS := +program_LIBRARIES := +CPPFLAGS += -g -std=c++11 -pthread -O3 + +CPPFLAGS += $(foreach includedir,$(program_INCLUDE_DIRS),-I$(includedir)) +LDFLAGS += $(foreach librarydir,$(program_LIBRARY_DIRS),-L$(librarydir)) +LDFLAGS += $(foreach library,$(program_LIBRARIES),-l$(library)) +LDFLAGS += -lboost_program_options + +CC=g++ + +.PHONY: all clean distclean + +all: $(program_NAME) + +$(program_NAME): $(program_OBJS) + $(CC) $(CPPFLAGS) $(program_OBJS) -o $(program_NAME) $(LDFLAGS) + +clean: + @- $(RM) $(program_NAME) + @- $(RM) $(program_OBJS) + +parser: + $(MAKE) -C ../../api/lexyacc + cp ../../api/lexyacc/smc_parser . + +distclean: clean diff --git a/sources/apps/SanityCheck/sanity.cpp b/sources/apps/SanityCheck/sanity.cpp new file mode 100644 index 0000000..0c7b0f2 --- /dev/null +++ b/sources/apps/SanityCheck/sanity.cpp @@ -0,0 +1,294 @@ +#include <iostream> + +#include <boost/program_options.hpp> + +#include "instruction.h" +#include "prog.h" +#include "platform.h" + +using namespace std; +namespace po = boost::program_options; + +#define USE_SMC_FILE 0 + +#define NUM_ROWS_REG 8 +#define NUM_BANKS_REG 11 +#define CASR 0 +#define BASR 1 +#define RASR 2 + +#define PATTERN_REG 12 + +#define BAR 7 +#define RAR 6 +#define CAR 4 + +#define NUM_COLS_REG 14 +#define LOOP_COLS 13 + +#define TEMP_PATTERN_REG 15 + +/** + * @return an instruction formed by NOPs + */ +Inst all_nops() +{ + return __pack_mininsts(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()); +} + +int main(int argc, char* argv[]) +{ + uint num_banks, num_rows, num_cols; + + po::options_description help("General options"); + help.add_options()("help,h", "Display the help message\n"); + + po::options_description dram_org("DRAM organization options"); + dram_org.add_options() + ("num_banks", po::value<uint>(&num_banks)->required(), "Specify the number of banks in the chip") + ("num_rows", po::value<uint>(&num_rows )->required(), "Specify the number of rows per bank") + ("num_cols", po::value<uint>(&num_cols )->required(), "Specify the number of columns (cachelines) per row\n") + ; + + po::options_description all; + all.add(help).add(dram_org); + + po::variables_map vm; + try + { + po::store(po::parse_command_line(argc, argv, all), vm); + po::notify(vm); + } + catch(const po::error& e) + { + if (vm.count("help") || argc == 1) + { + std::cout << all << "\n"; + std::exit(1); + } + else + { + std::cerr << "CRITICAL: " << e.what() << std::endl; + std::cerr << " Use \"--help\" to view all options" << std::endl; + std::exit(1); + } + } + + SoftMCPlatform platform; + int err; + uint32_t wr_pattern; + // buffer allocated for reading data from the board + char* buf = (char*) malloc(sizeof(char)*32*1024); + + // Initialize the platform, opens file descriptors for the board PCI-E interface. + if((err = platform.init()) != SOFTMC_SUCCESS) + { + cerr << "Could not initialize SoftMC Platform: " << err << endl; + exit(-1); + } + // reset the board to hopefully restore the board's state + platform.reset_fpga(); + + Program program; + /* SoftMC programs are formed sequentially by adding + * instructions to the program one by one. Instructions + * can be added to the program using add_inst(). + */ + + program.add_inst(SMC_LI(num_rows, NUM_ROWS_REG)); // load NUM_ROWS into NUM_ROWS_REG + program.add_inst(SMC_LI(num_banks, NUM_BANKS_REG)); // load NUM_BANKS into NUM_BANKS_REG + + // Stride registers are used to increment the row,bank,column registers without + // using regular (non-DDR) instructions. This way the DRAM array can be traversed much faster + + program.add_inst(SMC_LI(8, CASR)); // Load 8 into CASR since each READ reads 8 columns + program.add_inst(SMC_LI(1, BASR)); // Load 1 into BASR + program.add_inst(SMC_LI(1, RASR)); // Load 1 into RASR + + // Initialize PRNG, this test will randomly generate + // the first data to write into DRAM, it will modify + // the data by cyclic shifting and multiplying it with itself + // during runtime (this will be further explained down below). + srand(time(NULL)+getpid()); + uint16_t rand1 = (uint16_t) rand(); + wr_pattern = rand1 << 16; + wr_pattern |= ((uint16_t)rand()); + printf("wr_pattern: %x\n", wr_pattern); + + // Load WR data into pattern register whose content + // will be replicated among all words in the wide register + // Wide register is a 512-bit register, its content will + // be written to DRAM with SMC_WR commands. + program.add_inst(SMC_LI(wr_pattern, PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + program.add_inst(SMC_LDWD(PATTERN_REG,i)); + + program.add_inst(SMC_LI(0, BAR)); // Initialize BAR (bank address register) with 0 + + // add_label is used to add a label that can be used as a branch target to the program + program.add_label("BANK_BEGIN"); + + program.add_inst(SMC_LI(0, RAR)); // Initialize RAR with 0 + + program.add_label("ROW_BEGIN"); + + program.add_inst(SMC_LI(0, CAR)); // Initialize CAR with 0 + + // Four DRAM commands are issued at the same time + // to DRAM by SoftMC. + + // PREcharge bank BAR and wait for tRP (11 NOPs, 11 SoftMC cycles) + program.add_inst( + SMC_PRE(BAR, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + + // ACT & wait for tRCD + program.add_inst( + SMC_ACT(BAR, 0, RAR, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + + // We are now going to loop over all 512-bits in the row + // Since each row is 2^13 bytes we need to WR 128 times + program.add_inst(SMC_LI(128,NUM_COLS_REG)); // Load COL_SIZE register + program.add_inst(SMC_LI(0,LOOP_COLS)); // Load loop variable + + /* + * Try to write a different pattern to each cache line + * in DRAM. + */ + program.add_label("WR_BEGIN"); + + // Write to a whole row, increment CAR per WR + program.add_inst( + SMC_WRITE(BAR, 0, CAR, 1, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + + // Reload wide reg with the new data pattern + program.add_inst(SMC_SRC(PATTERN_REG,PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + program.add_inst(SMC_LDWD(PATTERN_REG,i)); + + program.add_inst(SMC_ADDI(LOOP_COLS,1,LOOP_COLS)); + // add a branch lower instruction which will jump to WR_BEGIN + // as long as LOOP_COLS<NUM_COLS, to iterate over the whole row + program.add_branch(program.BR_TYPE::BL,LOOP_COLS,NUM_COLS_REG, "WR_BEGIN"); + // END "WR_BEGIN" LOOP + + // Add more randomness to the PATTERN_REG by multiplying it by 3 + // This happens when a row is written to + program.add_inst(SMC_MV(PATTERN_REG,TEMP_PATTERN_REG)); + program.add_inst(SMC_ADD(PATTERN_REG,TEMP_PATTERN_REG,PATTERN_REG)); + program.add_inst(SMC_ADD(PATTERN_REG,TEMP_PATTERN_REG,PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + program.add_inst(SMC_LDWD(PATTERN_REG,i)); + + // Wait for t(write-precharge) + // & precharge the open bank + program.add_inst(all_nops()); + program.add_inst(all_nops()); + program.add_inst( + SMC_PRE(BAR, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + + program.add_inst(SMC_LI(0, CAR)); // reload CAR we are now going to read the whole row + + // PRE & wait for tRP + program.add_inst( + SMC_PRE(BAR, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + + // ACT & wait for tRCD + program.add_inst( + SMC_ACT(BAR, 0, RAR, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + + // Read from a whole row + // Essentially this can be imagined as an unrolled read whole row loop + for(int i = 0 ; i < num_cols ; i++){ + program.add_inst( + SMC_READ(BAR, 0, CAR, 1, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + program.add_inst(all_nops()); + } + + // Wait for t(read-precharge) + // & precharge the open bank + program.add_inst(all_nops()); + program.add_inst(all_nops()); + program.add_inst( + SMC_PRE(BAR, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + + // Increment row address by one since we are finished with this row + program.add_inst(SMC_ADDI(RAR,1,RAR)); + program.add_branch(program.BR_TYPE::BL,RAR,NUM_ROWS_REG, "ROW_BEGIN"); + program.add_inst(SMC_ADDI(BAR,1,BAR)); + program.add_branch(program.BR_TYPE::BL,BAR,NUM_BANKS_REG,"BANK_BEGIN"); + + program.add_inst(all_nops()); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + + // Tells SoftMC that the user program has ended + program.add_inst(SMC_END()); + + // Transfer the program to the FPGA board + platform.execute(program); + + // Read count + int rc = 0; + + uint32_t pattern = wr_pattern; + uint64_t err_count = 0; + // Below loop reads continuously from the FPGA board + // since we know in what order we are issuing our reads + // we know what physical address they are reading + while(1) + { + platform.receiveData(buf, 64 * num_cols); // read one row each iteration + for(int j = 0 ; j < num_cols ; j++){ + // printf("Whole pattern for cacheline %d: %x\n", j, pattern); + for (int k = 0 ; k < 64 ; k++){ // each byte in cache line + uint8_t mini_patt = pattern >> ((uint32_t)(k%4)*8); + if(mini_patt != (uint8_t)buf[j*64 + k]){ + err_count++; + fprintf(stderr,"t1: Pattern mismatch! Bank: %d Row: %d CacheLine: %d Byte: %d Expect: %x Read: %x\n", + rc/(num_rows), rc%(num_rows), j, k, mini_patt, (uint8_t)buf[j*64 + k]); + } + } + + //shift pattern at each row iteration + uint32_t imd = pattern >> (uint32_t)1; + uint32_t imd2 = pattern << (uint32_t)31; + pattern = imd | imd2; + } + pattern = pattern*(uint32_t)3; + rc++; + if(rc % (num_rows) == 0) + printf("Bank %d finished!\n", rc/num_rows); + if(rc == num_rows*num_banks) + break; + } + + if (err_count != 0) + exit(-1); + else + return 0; +} diff --git a/sources/apps/Smalltest/.gitignore b/sources/apps/Smalltest/.gitignore new file mode 100644 index 0000000..57855ab --- /dev/null +++ b/sources/apps/Smalltest/.gitignore @@ -0,0 +1 @@ +SoftMC_rdwr diff --git a/sources/apps/Smalltest/Makefile b/sources/apps/Smalltest/Makefile new file mode 100755 index 0000000..c7498df --- /dev/null +++ b/sources/apps/Smalltest/Makefile @@ -0,0 +1,32 @@ +program_NAME := SoftMC_rdwr +program_CXX_SRCS := read_write.cpp $(wildcard ../../api/*.c) $(wildcard ../../api/*.cpp) +program_CXX_OBJS := ${program_CXX_SRCS:.cpp=.o} +program_CXX_OBJS := ${program_CXX_OBJS:.c=.o} +program_OBJS := $(program_CXX_OBJS) +program_INCLUDE_DIRS := ../../api ../../../boost-lib +program_LIBRARY_DIRS := +program_LIBRARIES := +CPPFLAGS += -g -std=c++11 -pthread -O3 + +CPPFLAGS += $(foreach includedir,$(program_INCLUDE_DIRS),-I$(includedir)) +LDFLAGS += $(foreach librarydir,$(program_LIBRARY_DIRS),-L$(librarydir)) +LDFLAGS += $(foreach library,$(program_LIBRARIES),-l$(library)) + +CC=g++ + +.PHONY: all clean distclean + +all: $(program_NAME) + +$(program_NAME): $(program_OBJS) + $(CC) $(CPPFLAGS) $(program_OBJS) -o $(program_NAME) $(LDFLAGS) + +clean: + @- $(RM) $(program_NAME) + @- $(RM) $(program_OBJS) + +parser: + $(MAKE) -C ../../api/lexyacc + cp ../../api/lexyacc/smc_parser . + +distclean: clean diff --git a/sources/apps/Smalltest/read_write.cpp b/sources/apps/Smalltest/read_write.cpp new file mode 100644 index 0000000..608b17a --- /dev/null +++ b/sources/apps/Smalltest/read_write.cpp @@ -0,0 +1,293 @@ +#include "instruction.h" +#include "prog.h" +#include "platform.h" +#include <fstream> +#include <iostream> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <cstring> +#include <list> + +using namespace std; + +#define USE_SMC_FILE 0 + +#define NUM_BANKS 16 +#define NUM_ROWS 1024*32 + +#define NUM_ROWS_REG 8 +#define NUM_BANKS_REG 11 +// Stride register ids are fixed and should not be changed +// CASR should always be reg 0 +// BASR should always be reg 1 +// RASR should always be reg 2 +#define CASR 0 +#define BASR 1 +#define RASR 2 + +#define PATTERN_REG 12 + +#define BAR 7 +#define RAR 6 +#define CAR 4 + +#define NUM_COLS_REG 14 +#define LOOP_COLS 13 + +#define TEMP_PATTERN_REG 15 + +/** + * @return an instruction formed by NOPs + */ +Inst all_nops() +{ + return __pack_mininsts(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP()); +} + +int main() +{ + SoftMCPlatform platform; + int err; + uint32_t wr_pattern; + // buffer allocated for reading data from the board + char* buf = (char*) malloc(sizeof(char)*32*1024); + + // Demo using the smc file as program input + if(USE_SMC_FILE) + { + // Load demo smc program from file using + // program's constructor + Program program("read_write.smc"); + wr_pattern = 0x1a2b3c4d; + printf("Program Ready\n"); + + // Initialize the platform, opens file descriptors for the board PCI-E interface. + if((err = platform.init()) != SOFTMC_SUCCESS){ + cerr << "Could not initialize SoftMC Platform: " << err << endl; + } + // reset the board to hopefully restore the board's state + platform.reset_fpga(); + // Transfer the program to the FPGA board + platform.execute(program); + printf("Sent instructions\n"); + } + else + { + + // Initialize the platform, opens file descriptors for the board PCI-E interface. + if((err = platform.init()) != SOFTMC_SUCCESS){ + cerr << "Could not initialize SoftMC Platform: " << err << endl; + } + // reset the board to hopefully restore the board's state + platform.reset_fpga(); + + Program program; + /* SoftMC programs are formed sequentially by adding + * instructions to the program one by one. Instructions + * can be added to the program using add_inst(). + */ + + program.add_inst(SMC_LI(NUM_ROWS, NUM_ROWS_REG)); // load NUM_ROWS into NUM_ROWS_REG + program.add_inst(SMC_LI(NUM_BANKS, NUM_BANKS_REG)); // load NUM_BANKS into NUM_BANKS_REG + + // Stride registers are used to increment the row,bank,column registers without + // using regular (non-DDR) instructions. This way the DRAM array can be traversed much faster + + program.add_inst(SMC_LI(8, CASR)); // Load 8 into CASR since each READ reads 8 columns + program.add_inst(SMC_LI(1, BASR)); // Load 1 into BASR + program.add_inst(SMC_LI(1, RASR)); // Load 1 into RASR + + // Initialize PRNG, this test will randomly generate + // the first data to write into DRAM, it will modify + // the data by cyclic shifting and multiplying it with itself + // during runtime (this will be further explained down below). + srand(time(NULL)+getpid()); + uint16_t rand1 = (uint16_t) rand(); + wr_pattern = rand1 << 16; + wr_pattern |= ((uint16_t)rand()); + printf("wr_pattern: %x\n", wr_pattern); + + // Load WR data into pattern register whose content + // will be replicated among all words in the wide register + // Wide register is a 512-bit register, its content will + // be written to DRAM with SMC_WR commands. + program.add_inst(SMC_LI(wr_pattern, PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + program.add_inst(SMC_LDWD(PATTERN_REG,i)); + + program.add_inst(SMC_LI(0, BAR)); // Initialize BAR (bank address register) with 0 + + // add_label is used to add a label that can be used as a branch target to the program + program.add_label("BANK_BEGIN"); + + program.add_inst(SMC_LI(0, RAR)); // Initialize RAR with 0 + + program.add_label("ROW_BEGIN"); + + program.add_inst(SMC_LI(0, CAR)); // Initialize CAR with 0 + + // Four DRAM commands are issued at the same time + // to DRAM by SoftMC. + + // PREcharge bank BAR and wait for tRP (11 NOPs, 11 SoftMC cycles) + program.add_inst( + SMC_PRE(BAR, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + + // ACT & wait for tRCD + program.add_inst( + SMC_ACT(BAR, 0, RAR, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + + // We are now going to loop over all 512-bits in the row + // Since each row is 2^13 bytes we need to WR 128 times + program.add_inst(SMC_LI(128,NUM_COLS_REG)); // Load COL_SIZE register + program.add_inst(SMC_LI(0,LOOP_COLS)); // Load loop variable + + /* + * Try to write a different pattern to each cache line + * in DRAM. + */ + program.add_label("WR_BEGIN"); + + // Write to a whole row, increment CAR per WR + program.add_inst( + SMC_WRITE(BAR, 0, CAR, 1, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + + // Reload wide reg with the new data pattern + program.add_inst(SMC_SRC(PATTERN_REG,PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + program.add_inst(SMC_LDWD(PATTERN_REG,i)); + + program.add_inst(SMC_ADDI(LOOP_COLS,1,LOOP_COLS)); + // add a branch lower instruction which will jump to WR_BEGIN + // as long as LOOP_COLS<NUM_COLS, to iterate over the whole row + program.add_branch(program.BR_TYPE::BL,LOOP_COLS,NUM_COLS_REG, "WR_BEGIN"); + // END "WR_BEGIN" LOOP + + // Add more randomness to the PATTERN_REG by multiplying it by 3 + // This happens when a row is written to + program.add_inst(SMC_MV(PATTERN_REG,TEMP_PATTERN_REG)); + program.add_inst(SMC_ADD(PATTERN_REG,TEMP_PATTERN_REG,PATTERN_REG)); + program.add_inst(SMC_ADD(PATTERN_REG,TEMP_PATTERN_REG,PATTERN_REG)); + for(int i = 0 ; i < 16 ; i++) + program.add_inst(SMC_LDWD(PATTERN_REG,i)); + + // Wait for t(write-precharge) + // & precharge the open bank + program.add_inst(all_nops()); + program.add_inst(all_nops()); + program.add_inst( + SMC_PRE(BAR, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + + program.add_inst(SMC_LI(0, CAR)); // reload CAR we are now going to read the whole row + + // PRE & wait for tRP + program.add_inst( + SMC_PRE(BAR, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + + // ACT & wait for tRCD + program.add_inst( + SMC_ACT(BAR, 0, RAR, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + + // Read from a whole row + // Essentially this can be imagined as an unrolled read whole row loop + for(int i = 0 ; i < 128 ; i++){ + program.add_inst( + SMC_READ(BAR, 0, CAR, 1, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + program.add_inst(all_nops()); + } + + // Wait for t(read-precharge) + // & precharge the open bank + program.add_inst(all_nops()); + program.add_inst(all_nops()); + program.add_inst( + SMC_PRE(BAR, 0, 0), + SMC_NOP(), SMC_NOP(), SMC_NOP() + ); + + // Increment row address by one since we are finished with this row + program.add_inst(SMC_ADDI(RAR,1,RAR)); + program.add_branch(program.BR_TYPE::BL,RAR,NUM_ROWS_REG, "ROW_BEGIN"); + program.add_inst(SMC_ADDI(BAR,1,BAR)); + program.add_branch(program.BR_TYPE::BL,BAR,NUM_BANKS_REG,"BANK_BEGIN"); + + program.add_inst(all_nops()); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + program.add_inst(all_nops()); + + program.dump_registers(); + + // Tells SoftMC that the user program has ended + program.add_inst(SMC_END()); + + // Transfer the program to the FPGA board + platform.execute(program); + printf("Program Ready\n"); + program.pretty_print(); + printf("Sent instructions\n"); + } + + // Read count + int rc = 0; + + uint32_t pattern = wr_pattern; + uint64_t err_count = 0; + // Below loop reads continuously from the FPGA board + // since we know in what order we are issuing our reads + // we know what physical address they are reading + while(1) + { + platform.receiveData(buf, 8*1024); // read one row each iteration + for(int j = 0 ; j < 128 ; j++){ + // printf("Whole pattern for cacheline %d: %x\n", j, pattern); + for (int k = 0 ; k < 64 ; k++){ // each byte in cache line + uint8_t mini_patt = pattern >> ((uint32_t)(k%4)*8); + if(mini_patt != (uint8_t)buf[j*64 + k]){ + err_count++; + fprintf(stderr,"t1: Pattern mismatch! Bank: %d Row: %d CacheLine: %d Byte: %d Expect: %x Read: %x\n", + rc/(1024*32), rc%(1024*32), j, k, mini_patt, (uint8_t)buf[j*64 + k]); + //fprintf(stderr,"%d %d %d %x\n", + // rc/(1024*32), rc%(1024*32), j*64 + k, mini_patt^(uint8_t)buf[j*64 + k]); + } + } + + //shift pattern at each row iteration + uint32_t imd = pattern >> (uint32_t)1; + uint32_t imd2 = pattern << (uint32_t)31; + pattern = imd | imd2; + } + pattern = pattern*(uint32_t)3; + rc++; + if(rc % (1024*32) == 0) + printf("Bank %d finished!\n", rc/1024/32); + if(rc == 1024*32*16) + break; + } + printf("%ld out of %ld bytes have errors, last pattern: 0x%x\n", + err_count, (uint64_t)NUM_BANKS*NUM_ROWS*8192, pattern); + platform.readRegisterDump(); +} diff --git a/sources/apps/Smalltest/read_write.smc b/sources/apps/Smalltest/read_write.smc new file mode 100644 index 0000000..c4bfd5f --- /dev/null +++ b/sources/apps/Smalltest/read_write.smc @@ -0,0 +1,170 @@ +// Initialize register ids +// Similar to #define +casr = 0 +basr = 1 +rasr = 2 +bar = 7 +rar = 6 +car = 4 +pattern_reg = 12 +no_row_reg = 8 +no_bank_reg = 11 +no_cols_reg = 14 +loop_cols_reg = 13 +temp_pattern_reg = 15 + +// Begin writing the program +LI no_row_reg 32768 // Write 2^15 to no_row_Reg +LI no_bank_reg 16 // write 16 to no_bank_reg + +// Set stride registers (not required if you +// are not going to increment address registers +// with ddr commands) +LI casr 8 // set column address stride to 8 +LI basr 1 // set bas to 1 +LI rasr 1 // set ras to 1 + +// Init pattern register, we are going to fill +// dram with some random pattern. +LI pattern_reg 0x1a2b3c4d + +// Load the data register with the pattern +// second argument here is the word offset +// we are writing to in the data register +// e.g. LDWD 0 pattern reg fills the first four bytes + +LDWD 0 pattern_reg +LDWD 1 pattern_reg +LDWD 2 pattern_reg +LDWD 3 pattern_reg +LDWD 4 pattern_reg +LDWD 5 pattern_reg +LDWD 6 pattern_reg +LDWD 7 pattern_reg +LDWD 8 pattern_reg +LDWD 9 pattern_reg +LDWD 10 pattern_reg +LDWD 11 pattern_reg +LDWD 12 pattern_reg +LDWD 13 pattern_reg +LDWD 14 pattern_reg +LDWD 15 pattern_reg + +LI bar 0 // Initialize bank address register + +BANK_BEGIN: // Add label to loop over banks +LI rar 0 // Reload rar per bank loop + +ROW_BEGIN: // Add label to loop over rows +LI car 0 // Reload car per row loop + +PRE bar // PREcharge bar, don't increment, no precharge-all +WAIT 11 // Insert seven cycles of NOPs + +ACT bar rar // ACTivate bar, rar +WAIT 11 + +// Each write will write a different pattern +// "car++" indicates that we will increment +// car by casr value. + +LI no_cols_reg 128 +LI loop_cols_reg 0 + +WR_BEGIN: + +WR bar car++ +WAIT 3 + +// Cyclic shift pattern register to add "randomness" +SRC pattern_reg pattern_reg + +// Reload the wide data register +LDWD 0 pattern_reg +LDWD 1 pattern_reg +LDWD 2 pattern_reg +LDWD 3 pattern_reg +LDWD 4 pattern_reg +LDWD 5 pattern_reg +LDWD 6 pattern_reg +LDWD 7 pattern_reg +LDWD 8 pattern_reg +LDWD 9 pattern_reg +LDWD 10 pattern_reg +LDWD 11 pattern_reg +LDWD 12 pattern_reg +LDWD 13 pattern_reg +LDWD 14 pattern_reg +LDWD 15 pattern_reg + +// increment loop variable and iterate +ADDI loop_cols_reg loop_cols_reg 1 +BL WR_BEGIN loop_cols_reg no_cols_reg + +// multiply pattern_reg by 3 +MOV temp_pattern_reg pattern_reg +ADD pattern_reg pattern_reg temp_pattern_reg +ADD pattern_reg pattern_reg temp_pattern_reg + +// Reload the wide data register +LDWD 0 pattern_reg +LDWD 1 pattern_reg +LDWD 2 pattern_reg +LDWD 3 pattern_reg +LDWD 4 pattern_reg +LDWD 5 pattern_reg +LDWD 6 pattern_reg +LDWD 7 pattern_reg +LDWD 8 pattern_reg +LDWD 9 pattern_reg +LDWD 10 pattern_reg +LDWD 11 pattern_reg +LDWD 12 pattern_reg +LDWD 13 pattern_reg +LDWD 14 pattern_reg +LDWD 15 pattern_reg + +WAIT 8 +PRE bar +WAIT 3 + +// Reload car, we are now going to read from DRAM +LI car 0 + +// Precharge the open bank +PRE bar +WAIT 11 + +ACT bar rar +WAIT 11 + +// This is a basic primitive that can be used +// to replicate instructions-commands +// this block replicates the RD command 128 times +// and adds it to the program +times 128 { + RD bar car++ + WAIT 7 +} + +WAIT 8 +PRE bar +WAIT 3 + +// Increment rar per row loop +ADDI rar rar 1 +// if rar < no_row_reg, execute the "row loop" once more +BL ROW_BEGIN rar no_row_reg + +ADDI bar bar 1 +BL BANK_BEGIN bar no_bank_reg + +WAIT 16 + +// Tell softmc that the program has ended +// This is important! +END + +// Tell the parser the program definition has ended +// This is even more important!! +ENDPROGRAM diff --git a/sources/apps/parsertest/Makefile b/sources/apps/parsertest/Makefile new file mode 100755 index 0000000..6ca92f2 --- /dev/null +++ b/sources/apps/parsertest/Makefile @@ -0,0 +1,27 @@ +program_NAME := SoftMC_Example +program_CXX_SRCS := ptest.cpp $(wildcard ../../api/*.c*) +program_CXX_OBJS := ${program_CXX_SRCS:.cpp=.o} +program_OBJS := $(program_CXX_OBJS) +program_INCLUDE_DIRS := ../../api/ +program_LIBRARY_DIRS := +program_LIBRARIES := +CPPFLAGS += -g -std=c++11 + +CPPFLAGS += $(foreach includedir,$(program_INCLUDE_DIRS),-I$(includedir)) +LDFLAGS += $(foreach librarydir,$(program_LIBRARY_DIRS),-L$(librarydir)) +LDFLAGS += $(foreach library,$(program_LIBRARIES),-l$(library)) + +CC=g++ + +.PHONY: all clean distclean + +all: $(program_NAME) + +$(program_NAME): $(program_OBJS) + $(CC) $(CPPFLAGS) $(program_OBJS) -o $(program_NAME) $(LDFLAGS) + +clean: + @- $(RM) $(program_NAME) + @- $(RM) $(program_OBJS) + +distclean: clean diff --git a/sources/apps/parsertest/SoftMC_Example b/sources/apps/parsertest/SoftMC_Example Binary files differnew file mode 100755 index 0000000..c5146bf --- /dev/null +++ b/sources/apps/parsertest/SoftMC_Example diff --git a/sources/apps/parsertest/example.smc b/sources/apps/parsertest/example.smc new file mode 100644 index 0000000..a8cac7a --- /dev/null +++ b/sources/apps/parsertest/example.smc @@ -0,0 +1,9 @@ +LI 0 8; +LI 1 1; +LI 2 1; +LI 3 0; +LI 4 0; +LI 5 0; +BEGIN:; +WR 5 0 4 0 0 0, NOP, NOP, NOP; +END; diff --git a/sources/apps/parsertest/ptest.cpp b/sources/apps/parsertest/ptest.cpp new file mode 100644 index 0000000..28aabac --- /dev/null +++ b/sources/apps/parsertest/ptest.cpp @@ -0,0 +1,18 @@ +#include <string> +#include <fstream> +#include <iostream> +#include <sstream> +#include <string.h> +#include "prog.h" +#include "parser.h" + +int main() +{ + // https://stackoverflow.com/questions/2602013/read-whole-ascii-file-into-c-stdstring + std::ifstream t("example.smc"); + std::stringstream buffer; + buffer << t.rdbuf(); + string program = buffer.str(); + Parser p; + p.parse_program(program); +} diff --git a/sources/hdl/header_verilog/encoding.vh b/sources/hdl/header_verilog/encoding.vh new file mode 100644 index 0000000..f5d5697 --- /dev/null +++ b/sources/hdl/header_verilog/encoding.vh @@ -0,0 +1,103 @@ +// SoftMC instructions +`define FU_CODE_OFFSET 48 +`define BRANCH_OFFSET 62 +`define DDR_OFFSET 63 +`define INFO_OFFSET 61 +`define MEM_OFFSET 60 +`define BW_OFFSET 59 +`define DEC_RS1 0 +`define DEC_RS2 4 +`define DEC_IMD1 4 +`define DEC_IMD2 0 +`define DEC_IMD3 24 +`define DEC_RT 20 +`define DEC_WO 20 +`define DEC_JUMP_OFFSET 0 +`define DEC_SLEEP_OFFSET 0 +`define DEC_BR_TGT_OFFSET 8 +`define SR_OFFSET 56 +// DDR related +`define DDR_CODE_OFFSET 12 +`define DEC_CAR 4 +`define DEC_BAR 0 +`define DEC_RAR 4 +`define DEC_INC_BAR 10 +`define DEC_PRE_ALL 11 +`define DEC_INC_RAR 11 +`define DEC_INC_CAR 11 +`define DEC_AP 9 +`define DEC_BL4 8 +// function codes - exe +`define ADD 0 +`define ADDI 1 +`define SUB 2 +`define SUBI 3 +`define MV 4 +`define SRC 5 +`define LI 6 +`define LDWD 7 +`define LDPC 8 +`define SRE 0 //DDR command but no space left in ISA DDR instructions. +`define SRX 1 //DDR command but no space left in ISA DDR instructions. +`define BL 0 +`define BEQ 1 +`define JUMP 2 +`define SLEEP 3 +`define LD 0 +`define ST 1 +`define AND 0 +`define OR 1 +`define XOR 2 +// function codes - ddr +`define WRITE 0 +`define READ 1 +`define PRE 2 +`define ACT 3 +`define ZQ 4 +`define REF 5 +`define NOP 7 + +// SoftMC ddr uops +`define IS_WRITE 0 +`define IS_READ 1 +`define IS_PRE 2 +`define IS_ACT 3 +`define IS_ZQ 4 +`define IS_REF 5 +`define CAR 6 // column address register identifier +`define RAR 10 // row address register identifier +`define BAR 14 // bank address register identifier +`define PRE_ALL 18 +`define INC_CAR 19 // increment CAR after executing this +`define INC_RAR 20 +`define INC_BAR 21 +`define IS_NOP 22 +`define IS_BL4 23 +`define DO_AP 24 +`define IS_SRE 25 +`define IS_SRX 26 +// SoftMC exe uops +`define IS_ADD 0 +`define IS_SUB 1 +`define IS_MOV 2 +`define IS_LI 3 +`define IS_LDWD 4 +`define HAS_IMD 5 +`define IS_BL 6 +`define IS_BEQ 7 +`define IS_JUMP 8 +`define IS_SLEEP 9 +`define RS1 10 +`define RS2 14 +`define RT 18 +`define IMD 22 +`define IMD2 38 +`define IS_SRC 54 +`define IS_MEM 55 +`define IS_LD 56 +`define IS_ST 57 +`define IS_AND 58 +`define IS_OR 59 +`define IS_XOR 60 +`define IS_LDPC 61 + diff --git a/sources/hdl/header_verilog/parameters.vh b/sources/hdl/header_verilog/parameters.vh new file mode 100644 index 0000000..916c973 --- /dev/null +++ b/sources/hdl/header_verilog/parameters.vh @@ -0,0 +1,21 @@ +// Fetch +`define INSTR_WIDTH 64 +`define IMEM_ADDR_WIDTH 11 + +// Decode - Execute +`define DDR_UOP_WIDTH 27 +`define EXE_UOP_WIDTH 62 +`define BG_WIDTH 2 +`define BANK_WIDTH 2 +`define COL_WIDTH 10 +`define ROW_WIDTH 17 + +//Frontend +`define XDMA_AXI_DATA_WIDTH 256 +`define IMEM_RD_LATENCY 1 +//`define IMEM_SR // please only define when IMEM_RD_LATENCY > 1 + +//Common +`define HIGH 1'b1 +`define LOW 1'b0 + diff --git a/sources/hdl/sim_verilog/example_top.sv b/sources/hdl/sim_verilog/example_top.sv new file mode 100644 index 0000000..dbf4fea --- /dev/null +++ b/sources/hdl/sim_verilog/example_top.sv @@ -0,0 +1,557 @@ + +`ifdef MODEL_TECH + `ifndef CALIB_SIM + `define SIMULATION + `endif +`elsif INCA + `ifndef CALIB_SIM + `define SIMULATION + `endif +`elsif VCS + `ifndef CALIB_SIM + `define SIMULATION + `endif +`elsif XILINX_SIMULATOR + `ifndef CALIB_SIM + `define SIMULATION + `endif +`endif + +`timescale 1ps/1ps + +// Fetch +`define INSTR_WIDTH 64 +`define IMEM_ADDR_WIDTH 11 + +// Decode - Execute +`define DDR_UOP_WIDTH 27 +`define EXE_UOP_WIDTH 62 +`define BG_WIDTH 2 +`define BANK_WIDTH 2 +`define COL_WIDTH 10 +`define ROW_WIDTH 17 + +//Frontend +`define XDMA_AXI_DATA_WIDTH 256 +`define IMEM_RD_LATENCY 1 +//`define IMEM_SR // please only define when IMEM_RD_LATENCY > 1 + +//Common +`define HIGH 1'b1 +`define LOW 1'b0 + + + +`define UDIMM_x8 + +`define DQ_WIDTH 64 +`define ODT_WIDTH 2 +`define CS_WIDTH 2 +`define CKE_WIDTH 2 +`define CK_WIDTH 1 +`define ROW_ADDR_WIDTH 17 + + +module example_top # + ( + parameter SIMULATION = "FALSE" + ) + ( + + // common signals + input c0_sys_clk_p, + input c0_sys_clk_n, + input sys_rst, + + // iob <> ddr4 sdram ip signals + output c0_ddr4_act_n, + output [`ROW_ADDR_WIDTH-1:0] c0_ddr4_adr, + output [1:0] c0_ddr4_ba, + output [1:0] c0_ddr4_bg, + output [`CKE_WIDTH-1:0] c0_ddr4_cke, + output [`ODT_WIDTH-1:0] c0_ddr4_odt, + output [`CS_WIDTH-1:0] c0_ddr4_cs_n, + output [`CK_WIDTH-1:0] c0_ddr4_ck_t, + output [`CK_WIDTH-1:0] c0_ddr4_ck_c, + output c0_ddr4_reset_n, + + `ifdef RDIMM_x4 + inout [17:0] c0_ddr4_dqs_c, + inout [17:0] c0_ddr4_dqs_t, + inout [71:0] c0_ddr4_dq, + output c0_ddr4_parity, + `elsif UDIMM_x8 + inout [7:0] c0_ddr4_dqs_c, + inout [7:0] c0_ddr4_dqs_t, + inout [63:0] c0_ddr4_dq, + inout [7:0] c0_ddr4_dm_dbi_n, + output c0_ddr4_parity, + `elsif RDIMM_x8 + inout [8:0] c0_ddr4_dqs_c, + inout [8:0] c0_ddr4_dqs_t, + inout [71:0] c0_ddr4_dq, + inout [8:0] c0_ddr4_dm_dbi_n, + output c0_ddr4_parity, + `endif + + output c0_init_calib_complete, + output c0_data_compare_error +); + + `ifdef RDIMM_x4 + assign c0_ddr4_odt[1] = 1'b0; + assign c0_ddr4_cs_n[1] = 1'b1; + assign c0_ddr4_cke[1] = 1'b0; + `elsif RDIMM_x8 + assign c0_ddr4_odt[1] = 1'b0; + assign c0_ddr4_cs_n[1] = 1'b1; + assign c0_ddr4_cke[1] = 1'b0; + //assign c0_ddr4_parity = 1'b0; + `elsif UDIMM_x8 + assign c0_ddr4_odt[1] = 1'b0; + assign c0_ddr4_cs_n[1] = 1'b1; + assign c0_ddr4_cke[1] = 1'b0; + assign c0_ddr4_parity = 1'b0; + `endif + + // Frontend control signals + wire softmc_fin; + wire user_rst; + + // Frontend <-> Fetch signals + wire [`IMEM_ADDR_WIDTH-1:0] fr_addr_in; + wire fr_valid_in; + wire [`INSTR_WIDTH-1:0] fr_data_out; + wire fr_valid_out; + wire [`IMEM_ADDR_WIDTH-1:0] fr_addr_out; + wire fr_ready_out; + + // Frontend <-> misc. control signals + wire per_rd_init; + wire per_zq_init; + wire rbe_switch_mode; + + // AXI streaming ports + wire [`XDMA_AXI_DATA_WIDTH-1:0] m_axis_h2c_tdata_0,xdma_h2c_tdata_0; + wire m_axis_h2c_tlast_0, xdma_h2c_tlast_0; + wire m_axis_h2c_tvalid_0, xdma_h2c_tvalid_0; + wire m_axis_h2c_tready_0, xdma_h2c_tready_0; + wire [`XDMA_AXI_DATA_WIDTH/8-1:0] m_axis_h2c_tkeep_0, xdma_h2c_tkeep_0; + wire [`XDMA_AXI_DATA_WIDTH-1:0] s_axis_c2h_tdata_0, xdma_c2h_tdata_0; + wire s_axis_c2h_tlast_0, xdma_c2h_tlast_0; + wire s_axis_c2h_tvalid_0, xdma_c2h_tvalid_0; + wire s_axis_c2h_tready_0, xdma_c2h_tready_0; + wire [`XDMA_AXI_DATA_WIDTH/8-1:0] s_axis_c2h_tkeep_0, xdma_c2h_tkeep_0; + + // ddr_pipeline <-> outer module if + wire [3:0] ddr_write; + wire [3:0] ddr_read; + wire [3:0] ddr_pre; + wire [3:0] ddr_act; + wire [3:0] ddr_ref; + wire [3:0] ddr_zq; + wire [3:0] ddr_nop; + wire [3:0] ddr_sre; + wire [3:0] ddr_srx; + wire [3:0] ddr_ap; + wire [3:0] ddr_pall; + wire [3:0] ddr_half_bl; + wire [4*`BG_WIDTH-1:0] ddr_bg; + wire [4*`BANK_WIDTH-1:0] ddr_bank; + wire [4*`COL_WIDTH-1:0] ddr_col; + wire [4*`ROW_WIDTH-1:0] ddr_row; + wire [511:0] ddr_wdata; + + // periodic maintenance signals + wire ddr_maint_read; + + // phy <-> ddr adapter and dlltoggler signals + // dlltoggler + wire clk_sel = 0; + wire [7:0] dllt_mc_ACT_n; + wire [135:0] dllt_mc_ADR; + wire [15:0] dllt_mc_BA; + wire [15:0] dllt_mc_BG; + wire [7:0] dllt_mc_CKE; + wire [7:0] dllt_mc_CS_n; + wire dllt_done; + // adapter + wire [4:0] dBufAdr; + wire [`DQ_WIDTH*8-1:0] wrData; + wire [`DQ_WIDTH-1:0] wrDataMask; + wire [511:0] rdData; + wire [4:0] rdDataAddr; + wire [0:0] rdDataEn; + wire [0:0] rdDataEnd; + wire [0:0] per_rd_done; + wire [0:0] rmw_rd_done; + wire [4:0] wrDataAddr; + wire [0:0] wrDataEn; + wire [7:0] mc_ACT_n; + wire [135:0] mc_ADR; + wire [15:0] mc_BA; + wire [15:0] mc_BG; + wire [`CKE_WIDTH*8-1:0] mc_CKE; + wire [`CS_WIDTH*8-1:0] mc_CS_n; + wire [`ODT_WIDTH*8-1:0] mc_ODT; + wire [0:0] mcRdCAS; + wire [0:0] mcWrCAS; + wire [0:0] winInjTxn; + wire [0:0] winRmw; + wire [4:0] winBuf; + wire [1:0] winRank; + wire [5:0] tCWL; + wire dbg_clk; + wire c0_wr_rd_complete; + wire c0_ddr4_clk; + wire c0_ddr4_dll_off_clk; + wire ddr4_ui_clk; + wire c0_ddr4_rst; + wire [511:0] dbg_bus; + wire [1:0] mcCasSlot; + wire mcCasSlot2; + wire gt_data_ready; + + wire read_seq_incoming; // next few instructions will read from DRAM + wire [11:0] incoming_reads; // how many reads next few instructions will issue + wire [11:0] buffer_space; // remaining buffer size + + // There is a possibility that these signals are on + // the critical path as observed in + // the previous iteration of SoftMC + reg c0_init_calib_complete_r, sys_rst_r; + wire iq_full, processing_iseq, rdback_fifo_empty; + reg dllt_active = 1'b0; + + reg toggle_dll = 1'b0; + reg dont = 1'b1; + + always @(posedge c0_ddr4_clk) begin + c0_init_calib_complete_r <= c0_init_calib_complete; + sys_rst_r <= sys_rst; + `ifdef ENABLE_DLL_TOGGLER + if(c0_init_calib_complete_r && dont) begin + toggle_dll <= 1'b1; + dont <= 1'b0; + end + if(toggle_dll) begin + dllt_active <= ~dllt_active; + toggle_dll <= 1'b0; + end + if(dllt_done) begin + dllt_active <= ~dllt_active; + end + `endif + end + + + `ifdef UDIMM_x8 + phy_ddr4_udimm phy_ddr4_i( + .sys_rst (sys_rst), + .c0_sys_clk_p (c0_sys_clk_p), + .c0_sys_clk_n (c0_sys_clk_n), + + `ifdef ENABLE_DLL_TOGGLER + .c0_ddr4_ui_clk (ddr4_ui_clk), + .addn_ui_clkout1 (c0_ddr4_dll_off_clk), + `else + .c0_ddr4_ui_clk (c0_ddr4_clk), + `endif + .c0_ddr4_ui_clk_sync_rst (c0_ddr4_rst), + .c0_init_calib_complete (c0_init_calib_complete), + .dbg_clk (dbg_clk), + .c0_ddr4_act_n (c0_ddr4_act_n), + .c0_ddr4_adr (c0_ddr4_adr), + .c0_ddr4_ba (c0_ddr4_ba), + .c0_ddr4_bg (c0_ddr4_bg), + .c0_ddr4_cke (c0_ddr4_cke), + .c0_ddr4_odt (c0_ddr4_odt), + .c0_ddr4_cs_n (c0_ddr4_cs_n), + .c0_ddr4_ck_t (c0_ddr4_ck_t), + .c0_ddr4_ck_c (c0_ddr4_ck_c), + .c0_ddr4_reset_n (c0_ddr4_reset_n), + //.ddr4_par (c0_ddr4_parity), + .c0_ddr4_dq (c0_ddr4_dq), + .c0_ddr4_dqs_c (c0_ddr4_dqs_c), + .c0_ddr4_dqs_t (c0_ddr4_dqs_t), + .c0_ddr4_dm_dbi_n (c0_ddr4_dm_dbi_n), + + .dBufAdr (dBufAdr), + .wrData (wrData), + .rdData (rdData), + .rdDataAddr (rdDataAddr), + .rdDataEn (rdDataEn), + .rdDataEnd (rdDataEnd), + .per_rd_done (per_rd_done), + .rmw_rd_done (rmw_rd_done), + .wrDataAddr (wrDataAddr), + .wrDataEn (wrDataEn), + .wrDataMask (wrDataMask), + + .mc_ACT_n (dllt_active ? dllt_mc_ACT_n : mc_ACT_n), + .mc_ADR (dllt_active ? dllt_mc_ADR : mc_ADR), + .mc_BA (dllt_active ? dllt_mc_BA : mc_BA), + .mc_BG (dllt_active ? dllt_mc_BG : mc_BG), + // DRAM CKE. 8 bits for each DRAM pin. The mc_CKE signal is always set to '1'. + .mc_CKE (dllt_active ? dllt_mc_CKE : {8{1'b1}}), + .mc_CS_n (dllt_active ? dllt_mc_CS_n : mc_CS_n), + .mc_ODT (mc_ODT), + // CAS command slot select. Slot0 is enabled for example design. + .mcCasSlot (dllt_active ? 0 : mcCasSlot), + // CAS slot 2 select. mcCasSlot2 serves a similar purpose as the mcCasSlot[1:0] signal, but mcCasSlot2 is used in timing + // critical logic in the Phy. Slot0 is enabled for example design. + .mcCasSlot2 (dllt_active ? 0 : mcCasSlot2), + .mcRdCAS (dllt_active ? 0 : mcRdCAS), + .mcWrCAS (dllt_active ? 0 : mcWrCAS), + // Optional read command type indication. The winInjTxn signal is set to '0' for example design. + .winInjTxn ({1{1'b0}}), + // Optional read command type indication. The winRmw signal is set to '0' for example design. + .winRmw ({1{1'b0}}), + // Update VT Tracking. The gt_data_ready signal is set to '0' in this example design. + // This signal must be asserted periodically to keep the DQS Gate aligned as voltage and temperature drift. + // For more information, Refer to PG150 document. + .gt_data_ready (gt_data_ready), + .winBuf (winBuf), + .winRank (winRank), + .tCWL (tCWL), + // Debug Port + .dbg_bus (dbg_bus) + ); + `endif + + `ifdef ENABLE_DLL_TOGGLER + //BUFGMUX:GeneralClockMuxBuffer + //UltraScale + //XilinxHDLLibrariesGuide, version2014.4 + BUFGMUX#(.CLK_SEL_TYPE("SYNC") //ASYNC,SYNC + )BUFGMUX_inst( + .O(c0_ddr4_clk), //1-bitoutput:Clockoutput + .I0(ddr4_ui_clk), //1-bitinput:Clockinput(S=0) + .I1(c0_ddr4_dll_off_clk), //1-bitinput:Clockinput(S=1) + .S(clk_sel) //1-bitinput:Clockselect + ); + //End of BUFGMUX_inst instantiation + `endif + + softmc_pipeline pipeline( + .clk(c0_ddr4_clk), + .rst(c0_ddr4_rst || user_rst || ~c0_init_calib_complete_r), + + .softmc_end(softmc_fin), + .read_size(incoming_reads), + .read_seq_incoming(read_seq_incoming), + .buffer_space(buffer_space), + + .addr_out(fr_addr_in), + .valid_out(fr_valid_in), + .data_in(fr_data_out), + .valid_in(fr_valid_out), + .addr_in(fr_addr_out), + .ready_out(fr_ready_out), + + .ddr_write(ddr_write), + .ddr_read(ddr_read), + .ddr_pre(ddr_pre), + .ddr_act(ddr_act), + .ddr_ref(ddr_ref), + .ddr_zq(ddr_zq), + .ddr_nop(ddr_nop), + .ddr_sre(ddr_sre), + .ddr_srx(ddr_srx), + .ddr_ap(ddr_ap), + .ddr_pall(ddr_pall), + .ddr_half_bl(ddr_half_bl), + .ddr_bg(ddr_bg), + .ddr_bank(ddr_bank), + .ddr_col(ddr_col), + .ddr_row(ddr_row), + .ddr_wdata(ddr_wdata) + ); + + wire frontend_ready; + + reg keep_frontend_reset = 1'b1; + + `ifdef ENABLE_DLL_TOGGLER + always @(posedge c0_ddr4_clk) begin + keep_frontend_reset <= c0_init_calib_complete_r; + end + `endif + + frontend #(.SIM_MEM("true"))frontend( + .clk(c0_ddr4_clk), + .rst(c0_ddr4_rst || ~keep_frontend_reset || ~c0_init_calib_complete_r || dllt_active), + + .init_calib_complete(c0_init_calib_complete_r), + .softmc_fin(softmc_fin), + .user_rst(user_rst), + //.dllt_begin(toggle_dll), + + // indicates read_back unit is ready for the next iseq + .frontend_ready(frontend_ready), + + // frontend <-> fetch stage if + .addr_in(fr_addr_in), + .valid_in(fr_valid_in), + .data_out(fr_data_out), + .valid_out(fr_valid_out), + .addr_out(fr_addr_out), + .ready_in(fr_ready_out), + + // frontend <-> xdma interface + .h2c_tdata_0(m_axis_h2c_tdata_0), + .h2c_tlast_0(m_axis_h2c_tlast_0), + .h2c_tvalid_0(m_axis_h2c_tvalid_0), + .h2c_tready_0(m_axis_h2c_tready_0), + .h2c_tkeep_0(m_axis_h2c_tkeep_0), + + .per_rd_init(per_rd_init), + .per_zq_init(per_zq_init), + .rbe_switch_mode(rbe_switch_mode) + ); + + ddr4_adapter #( + .DQ_WIDTH(`DQ_WIDTH) + ) ddr4_adapter + ( + .clk(c0_ddr4_clk), + .rst(c0_ddr4_rst || user_rst || ~c0_init_calib_complete_r), + .init_calib_complete(c0_init_calib_complete_r), + //.io_config_strobe, + //.io_config, + .dBufAdr(dBufAdr), // Reserved. Should be tied low. + .wrData(wrData), // DRAM write data. There are 8 bits for each DQ lane on the DRAM bus. + .wrDataMask(wrDataMask),// DRAM write DM/DBI port.There is one bit for each byte of the wrData port. + .wrDataEn(wrDataEn), // Write data Enable. The Phy will assert this port for one cycle for each write CAS command. + .mc_ACT_n(mc_ACT_n), // DRAM ACT_n command signal for four DRAM clock cycles. + .mc_ADR(mc_ADR), // DRAM address. There are 8 bits in the fabric interface for each address bit on the DRAM bus. + .mc_BA(mc_BA), // DRAM bank address. 8 bits for each DRAM bank address. + .mc_BG(mc_BG), // DRAM bank group address. + .mc_CS_n(mc_CS_n), // DRAM CS_n + //.mc_CKE(mc_CKE), // DRAM CKE + //.mc_ODT(mc_ODT), // DRAM ODT + .mcRdCAS(mcRdCAS), // Read CAS command issued. + .mcWrCAS(mcWrCAS), // Write CAS command issued. + .winRank(winRank), // Target rank for CAS commands. This value indicates which rank a CAS command is issued to. + .winBuf(winBuf), // Optional control signal. When either mcRdCAS or mcWrCAS is asserted, the Phy will store the value on the winBuf signal. + //.rdData(rdData), // DRAM read data. + .rdDataEn(rdDataEn), // Read data valid. This signal asserts for one fabric cycle for each completed read operation. + .rdDataEnd(rdDataEnd), // Unused. Tied high. + .mcCasSlot(mcCasSlot), + .mcCasSlot2(mcCasSlot2), + .gt_data_ready(gt_data_ready), + .ddr_write(ddr_write), + .ddr_read(ddr_read), + .ddr_pre(ddr_pre), + .ddr_act(ddr_act), + .ddr_ref(ddr_ref), + .ddr_zq(ddr_zq), + .ddr_nop(ddr_nop), + //.ddr_sre(ddr_sre), + //.ddr_srx(ddr_srx), + .ddr_ap(ddr_ap), + .ddr_pall(ddr_pall), + .ddr_half_bl(ddr_half_bl), + .ddr_bg(ddr_bg), + .ddr_bank(ddr_bank), + .ddr_col(ddr_col), + .ddr_row(ddr_row), + .ddr_wdata(ddr_wdata), + + .ddr_maint_read(per_rd_init) + ); + + localparam ODTWRDEL = 5'd9; + localparam ODTWRDUR = 4'd6; + localparam ODTWRODEL = 5'd9; + localparam ODTWRODUR = 4'd6; + localparam ODTRDDEL = 5'd10; + localparam ODTRDDUR = 4'd6; + localparam ODTRDODEL = 5'd9; + localparam ODTRDODUR = 4'd6; + localparam ODTNOP = 16'h0000; + localparam ODTWR = 16'h0001; + localparam ODTRD = 16'h0000; + + + wire tranSentC; + assign tranSentC = mcRdCAS | mcWrCAS; + + //synthesis translate_on + //******************************************************************************* + ddr4_mc_odt # ( + .ODTWR (ODTWR) + ,.ODTWRDEL (ODTWRDEL) + ,.ODTWRDUR (ODTWRDUR) + ,.ODTWRODEL (ODTWRODEL) + ,.ODTWRODUR (ODTWRODUR) + + ,.ODTRD (ODTRD) + ,.ODTRDDEL (ODTRDDEL) + ,.ODTRDDUR (ODTRDDUR) + ,.ODTRDODEL (ODTRDODEL) + ,.ODTRDODUR (ODTRDODUR) + + ,.ODTNOP (ODTNOP) + ,.ODTBITS (`ODT_WIDTH) + ,.TCQ (0.1) + )u_ddr_tb_odt( + .clk (c0_ddr4_clk) + ,.rst (c0_ddr4_rst) + ,.mc_ODT (mc_ODT) + ,.casSlot (mcCasSlot) + ,.casSlot2 (mcCasSlot2) + ,.rank (winRank) + ,.winRead (mcRdCAS) + ,.winWrite (mcWrCAS) + ,.tranSentC (tranSentC) + ); + + readback_engine rbe( + + // common signals + .clk(c0_ddr4_clk), + .rst(c0_ddr4_rst || user_rst || ~c0_init_calib_complete_r), + + // other ctrl signals + .flush(frontend_ready), + .switch_mode(rbe_switch_mode), + .read_seq_incoming(read_seq_incoming), // next few instructions will read from DRAM + .incoming_reads(incoming_reads), // how many reads next few instructions will issue + .buffer_space(buffer_space), // remaining buffer size + // DRAM <-> engine if + .rd_data(rdData), + .rd_valid(rdDataEn), + + // rbe <-> rf interface + .ddr_wdata(ddr_wdata), + + .per_rd_init(per_rd_init), + .per_zq_init(per_zq_init), + + // rbe <-> xdma if + .c2h_tdata_0(s_axis_c2h_tdata_0), + .c2h_tlast_0(s_axis_c2h_tlast_0), + .c2h_tvalid_0(s_axis_c2h_tvalid_0), + .c2h_tready_0(1'b1), + .c2h_tkeep_0(s_axis_c2h_tkeep_0) + + ); + + `ifdef ENABLE_DLL_TOGGLER + dll_toggler dllt + ( + .clk(c0_ddr4_clk), + .rst(c0_ddr4_rst || user_rst || ~c0_init_calib_complete_r), + .toggle_valid(toggle_dll), + .dllt_done(dllt_done), + .mc_ACT_n(dllt_mc_ACT_n), // DRAM ACT_n command signal for four DRAM clock cycles. + .mc_ADR(dllt_mc_ADR), // DRAM address. There are 8 bits in the fabric interface for each address bit on the DRAM bus. + .mc_BA(dllt_mc_BA), // DRAM bank address. 8 bits for each DRAM bank address. + .mc_BG(dllt_mc_BG), // DRAM bank group address. + .mc_CS_n(dllt_mc_CS_n), // DRAM CS_n + .mc_CKE(dllt_mc_CKE), + .clk_sel(clk_sel) + ); + `endif +endmodule diff --git a/sources/hdl/sim_verilog/tb_decode_stage.v b/sources/hdl/sim_verilog/tb_decode_stage.v new file mode 100644 index 0000000..1842cf7 --- /dev/null +++ b/sources/hdl/sim_verilog/tb_decode_stage.v @@ -0,0 +1,26 @@ +module tb_decode_stage( + + ); + + reg [63:0] instr; + reg [11:0] instr_pc; + reg instr_valid; + + decode_stage ds( + .instr(instr), + .instr_pc(instr_pc), + .instr_valid(instr_valid) + ); + + reg clk = 0, rst = 1; + + initial begin + #100; + rst = 0; + end + + always begin + #5; + clk = ~clk; + end +endmodule diff --git a/sources/hdl/sim_verilog/tb_frontend.v b/sources/hdl/sim_verilog/tb_frontend.v new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/sources/hdl/sim_verilog/tb_frontend.v diff --git a/sources/hdl/sim_verilog/tb_softmc_top.v b/sources/hdl/sim_verilog/tb_softmc_top.v new file mode 100644 index 0000000..6a61f44 --- /dev/null +++ b/sources/hdl/sim_verilog/tb_softmc_top.v @@ -0,0 +1,42 @@ +`timescale 1ns / 1ps +////////////////////////////////////////////////////////////////////////////////// +// Company: +// Engineer: +// +// Create Date: 12/06/2018 05:43:09 PM +// Design Name: +// Module Name: tb_softmc_top +// Project Name: +// Target Devices: +// Tool Versions: +// Description: +// +// Dependencies: +// +// Revision: +// Revision 0.01 - File Created +// Additional Comments: +// +////////////////////////////////////////////////////////////////////////////////// + + +module tb_only_softmc( + + ); + + reg clk = 0, rst = 1; + softmc_top smct( + .clk(clk), + .rst(rst) + ); + + initial begin + #50; + rst = 0; + end + + always begin + #5; + clk = ~clk; + end +endmodule diff --git a/sources/hdl/verilog/ddr_pipeline.v b/sources/hdl/verilog/ddr_pipeline.v new file mode 100644 index 0000000..f8ae5aa --- /dev/null +++ b/sources/hdl/verilog/ddr_pipeline.v @@ -0,0 +1,206 @@ +`include "parameters.vh" +`include "encoding.vh" + +module ddr_pipeline( + + // common signals + input clk, + input rst, + + // execute_stage <-> ddr_pipeline if + input ddr_valid, + input [`DDR_UOP_WIDTH*4-1:0] ddr_uop, + + // ddr_pipeline <-> outer DDRX IP interface + output [3:0] ddr_write, + output [3:0] ddr_read, + output [3:0] ddr_pre, + output [3:0] ddr_act, + output [3:0] ddr_ref, + output [3:0] ddr_zq, + output [3:0] ddr_nop, + output [3:0] ddr_sre, + output [3:0] ddr_srx, + output [3:0] ddr_ap, + output [3:0] ddr_pall, + output [3:0] ddr_half_bl, + output [4*`BG_WIDTH-1:0] ddr_bg, + output [4*`BANK_WIDTH-1:0] ddr_bank, + output [4*`COL_WIDTH-1:0] ddr_col, + output [4*`ROW_WIDTH-1:0] ddr_row, + output [511:0] ddr_wdata, + + // ddr_pipeline <-> regfile interface + input [511:0] wide_reg, + output [7:0] update_en, + output [4*8-1:0] update_ids, + output [32*8-1:0] update_vals, + input [`COL_WIDTH-1:0] casr, + input [`BANK_WIDTH+`BG_WIDTH-1:0] basr, + input [`ROW_WIDTH-1:0] rasr, + output [4*8-1:0] reg_ids, // registers we need to read + input [32*8-1:0] reg_vals // register values + ); + + //Split input uop into four ways + wire [`DDR_UOP_WIDTH-1:0] uop [3:0]; + genvar uops; + generate + for(uops = 0 ; uops < 4 ; uops = uops + 1) begin: split_uops + assign uop[uops] = ddr_uop[`DDR_UOP_WIDTH*uops +: `DDR_UOP_WIDTH]; + end + endgenerate + + // Figure out reg ids in the bubble cycle + reg [4*8-1:0] reg_ids_ns, reg_ids_r; + + reg s2_valid; + reg [`DDR_UOP_WIDTH-1:0] s2_uop [3:0]; + reg [7:0] s2_update_en; // which registers will we update + reg [8*32-1:0] s2_update_val; + + reg [3:0] ddr_write_ns, ddr_write_r; + reg [3:0] ddr_read_ns, ddr_read_r; + reg [3:0] ddr_pre_ns, ddr_pre_r; + reg [3:0] ddr_act_ns, ddr_act_r; + reg [3:0] ddr_ref_ns, ddr_ref_r; + reg [3:0] ddr_sre_ns, ddr_sre_r; + reg [3:0] ddr_srx_ns, ddr_srx_r; + reg [3:0] ddr_zq_ns, ddr_zq_r; + reg [3:0] ddr_nop_ns, ddr_nop_r; + reg [3:0] ddr_ap_ns, ddr_ap_r; + reg [3:0] ddr_pall_ns, ddr_pall_r; + reg [3:0] ddr_half_bl_ns, ddr_half_bl_r; + reg [4*`BG_WIDTH-1:0] ddr_bg_ns, ddr_bg_r; + reg [4*`BANK_WIDTH-1:0] ddr_bank_ns, ddr_bank_r; + reg [4*`COL_WIDTH-1:0] ddr_col_ns, ddr_col_r; + reg [4*`ROW_WIDTH-1:0] ddr_row_ns, ddr_row_r; + reg [511:0] ddr_data_r; + + assign update_vals = s2_update_val; + assign update_en = s2_update_en; + assign update_ids = reg_ids_r; + assign reg_ids = reg_ids_r; + // these values are registered before + // being offloaded to the outer module + assign ddr_write = ddr_write_r; + assign ddr_read = ddr_read_r; + assign ddr_pre = ddr_pre_r; + assign ddr_act = ddr_act_r; + assign ddr_ref = ddr_ref_r; + assign ddr_sre = ddr_sre_r; + assign ddr_srx = ddr_srx_r; + assign ddr_zq = ddr_zq_r; + assign ddr_nop = ddr_nop_r; + assign ddr_ap = ddr_ap_r; + assign ddr_pall = ddr_pall_r; + assign ddr_half_bl = ddr_half_bl_r; + assign ddr_bg = ddr_bg_r; + assign ddr_bank = ddr_bank_r; + assign ddr_col = ddr_col_r; + assign ddr_row = ddr_row_r; + assign ddr_wdata = ddr_data_r; + + integer i; + always @* begin + ddr_nop_ns = {4{`HIGH}}; + s2_update_en = {4{`LOW}}; + reg_ids_ns = reg_ids_r; + s2_update_val = 8*32'bX; + for(i = 0 ; i < 4 ; i = i + 1) begin: gen_ddrx_sigs + // Decide which registers to read in stage 1 + if(uop[i][`IS_WRITE] || uop[i][`IS_READ]) begin + reg_ids_ns[i*8 +: 4] = uop[i][`BAR +: 4]; + reg_ids_ns[i*8+4 +: 4] = uop[i][`CAR +: 4]; + end + if(uop[i][`IS_PRE]) begin + reg_ids_ns[i*8 +: 4] = uop[i][`BAR +: 4]; + end + if(uop[i][`IS_ACT]) begin + reg_ids_ns[i*8 +: 4] = uop[i][`BAR +: 4]; + reg_ids_ns[i*8+4 +: 4] = uop[i][`RAR +: 4]; + end + // stage 2 control sigs + ddr_pall_ns[i] = s2_uop[i][`PRE_ALL] & s2_uop[i][`IS_PRE]; + ddr_write_ns[i] = s2_uop[i][`IS_WRITE]; + ddr_read_ns[i] = s2_uop[i][`IS_READ]; + ddr_pre_ns[i] = s2_uop[i][`IS_PRE]; + ddr_act_ns[i] = s2_uop[i][`IS_ACT]; + ddr_zq_ns[i] = s2_uop[i][`IS_ZQ]; + ddr_ref_ns[i] = s2_uop[i][`IS_REF]; + ddr_sre_ns[i] = s2_uop[i][`IS_SRE]; + ddr_srx_ns[i] = s2_uop[i][`IS_SRX]; + ddr_ap_ns[i] = s2_uop[i][`DO_AP] & (s2_uop[i][`IS_WRITE] | s2_uop[i][`IS_READ]); + ddr_half_bl_ns[i] = s2_uop[i][`IS_BL4] & (s2_uop[i][`IS_WRITE] | s2_uop[i][`IS_READ]); + ddr_nop_ns[i] = s2_valid ? s2_uop[i][`IS_NOP] : {4{`HIGH}}; + // stage 2 address calculation + ddr_row_ns[i*`ROW_WIDTH +: `ROW_WIDTH] + = reg_vals[i*64+32 +: `ROW_WIDTH]; + ddr_bank_ns[i*`BANK_WIDTH +: `BANK_WIDTH] + = reg_vals[i*64 +: `BANK_WIDTH]; + ddr_bg_ns[i*`BG_WIDTH +: `BG_WIDTH] + = reg_vals[i*64+`BANK_WIDTH +: `BG_WIDTH]; + ddr_col_ns[i*`COL_WIDTH +: `COL_WIDTH] + = reg_vals[i*64+32 +: `COL_WIDTH]; + // stage 2 reg update + if(s2_uop[i][`INC_CAR] & (s2_uop[i][`IS_WRITE] | s2_uop[i][`IS_READ])) begin + s2_update_en[i*2+1] = `HIGH; + s2_update_val[i*64+32 +: 32] = reg_vals[i*64+32 +: `COL_WIDTH] + + casr; + end + else if(s2_uop[i][`INC_RAR] & (s2_uop[i][`IS_ACT])) begin + s2_update_en[i*2+1] = `HIGH; + s2_update_val[i*64+32 +: 32] = reg_vals[i*64+32 +: `ROW_WIDTH] + + rasr; + end + if(s2_uop[i][`INC_BAR]) begin + s2_update_en[i*2] = `HIGH; + s2_update_val[i*64 +: 32] = reg_vals[i*64 +: `BANK_WIDTH+`BG_WIDTH] + + basr; + end + end // for + end + + always @(posedge clk) begin + if(rst) begin + s2_valid <= `LOW; + for(i = 0 ; i < 4 ; i = i + 1) + s2_uop[i] <= {`DDR_UOP_WIDTH{`LOW}}; + ddr_write_r <= {4{`LOW}}; + ddr_read_r <= {4{`LOW}}; + ddr_pre_r <= {4{`LOW}}; + ddr_act_r <= {4{`LOW}}; + ddr_ref_r <= {4{`LOW}}; + ddr_sre_r <= {4{`LOW}}; + ddr_srx_r <= {4{`LOW}}; + ddr_zq_r <= {4{`LOW}}; + ddr_nop_r <= {4{`HIGH}}; + ddr_ap_r <= {4{`LOW}}; + ddr_half_bl_r <= {4{`LOW}}; + end + else begin + reg_ids_r <= reg_ids_ns; + s2_valid <= ddr_valid; + for(i = 0 ; i < 4 ; i = i + 1) + s2_uop[i] <= uop[i]; + ddr_write_r <= ddr_write_ns; + ddr_read_r <= ddr_read_ns; + ddr_pre_r <= ddr_pre_ns; + ddr_act_r <= ddr_act_ns; + ddr_ref_r <= ddr_ref_ns; + ddr_sre_r <= ddr_sre_ns; + ddr_srx_r <= ddr_srx_ns; + ddr_zq_r <= ddr_zq_ns; + ddr_nop_r <= ddr_nop_ns; + ddr_ap_r <= ddr_ap_ns; + ddr_pall_r <= ddr_pall_ns; + ddr_half_bl_r <= ddr_half_bl_ns; + ddr_bg_r <= ddr_bg_ns; + ddr_bank_r <= ddr_bank_ns; + ddr_col_r <= ddr_col_ns; + ddr_row_r <= ddr_row_ns; + ddr_data_r <= wide_reg; + end + end + +endmodule diff --git a/sources/hdl/verilog/decode_stage.v b/sources/hdl/verilog/decode_stage.v new file mode 100644 index 0000000..7f40cba --- /dev/null +++ b/sources/hdl/verilog/decode_stage.v @@ -0,0 +1,318 @@ +`include "parameters.vh" +`include "encoding.vh" + +module decode_stage( + + input clk, + input rst, + + // fetch stage <-> decode stage interface + input [`INSTR_WIDTH-1:0] instr, + input [`IMEM_ADDR_WIDTH-1:0] instr_pc, + input instr_valid, + + // decode stage <-> execute stage interface + output ddr_valid, + output exe_valid, + output [`DDR_UOP_WIDTH*4-1:0] ddr_uop, + output [`EXE_UOP_WIDTH-1:0] exe_uop, + output [`IMEM_ADDR_WIDTH-1:0] exe_pc, + output [32*7-1:0] ddr_stat + ); + + reg ddr_valid_r, ddr_valid_ns; + reg exe_valid_r, exe_valid_ns; + reg [`IMEM_ADDR_WIDTH-1:0] exe_pc_r; + + reg [`DDR_UOP_WIDTH-1:0] ddr_uop_r [3:0], ddr_uop_ns [3:0]; + reg [`EXE_UOP_WIDTH-1:0] exe_uop_r, exe_uop_ns; + reg[31:0] ddr_stat_r[0:6]; //WRITE, READ, PRE, ACT, ZQ, REF, CYC counts + + // split the instr into four individual insts + localparam div = 16; + wire [`INSTR_WIDTH/4-1:0] ddr_insts [3:0]; + assign ddr_insts[0] = instr [0 +: div]; + assign ddr_insts[1] = instr [div +: div]; + assign ddr_insts[2] = instr [div*2 +: div]; + assign ddr_insts[3] = instr [div*3 +: div]; + // gather all ddr uops into one output (¯\_(ツ)_/¯) + // verilog does not let us define output vectors + genvar uops; + generate + for(uops = 0 ; uops < 4 ; uops = uops + 1) begin: gather_uops + assign ddr_uop[uops*`DDR_UOP_WIDTH +: `DDR_UOP_WIDTH] = + ddr_uop_r[uops]; + end + endgenerate + assign exe_uop = exe_uop_r; + assign exe_pc = exe_pc_r; + assign ddr_valid = ddr_valid_r; + assign exe_valid = exe_valid_r; + assign ddr_stat = {ddr_stat_r[6], ddr_stat_r[5], + ddr_stat_r[4], ddr_stat_r[3], + ddr_stat_r[2], ddr_stat_r[1], + ddr_stat_r[0]}; + + reg is_started = 0; + reg[26:0] imd; + integer i; + + always @* begin + ddr_valid_ns = `LOW; + exe_valid_ns = `LOW; + exe_uop_ns = {`EXE_UOP_WIDTH{`LOW}}; + for(i = 0 ; i < 4 ; i = i + 1) + ddr_uop_ns[i] = {`DDR_UOP_WIDTH{`LOW}}; + if(instr_valid) begin + // Decoding a DDR packet + if(instr[`DDR_OFFSET]) begin + ddr_valid_ns = `HIGH; + for(i = 0 ; i < 4 ; i = i + 1) begin: gen_ddr_uops + case(ddr_insts[i][`DDR_CODE_OFFSET +: 3]) + `WRITE: begin + ddr_uop_ns[i][`IS_WRITE] = `HIGH; + ddr_uop_ns[i][`INC_CAR] = ddr_insts[i][`DEC_INC_CAR]; + ddr_uop_ns[i][`INC_BAR] = ddr_insts[i][`DEC_INC_BAR]; + ddr_uop_ns[i][`CAR+:4] = ddr_insts[i][`DEC_CAR+:4]; + ddr_uop_ns[i][`BAR+:4] = ddr_insts[i][`DEC_BAR+:4]; + ddr_uop_ns[i][`DO_AP] = ddr_insts[i][`DEC_AP]; + ddr_uop_ns[i][`IS_BL4] = ddr_insts[i][`DEC_BL4]; + end + `READ: begin + ddr_uop_ns[i][`IS_READ] = `HIGH; + ddr_uop_ns[i][`INC_CAR] = ddr_insts[i][`DEC_INC_CAR]; + ddr_uop_ns[i][`INC_BAR] = ddr_insts[i][`DEC_INC_BAR]; + ddr_uop_ns[i][`CAR+:4] = ddr_insts[i][`DEC_CAR+:4]; + ddr_uop_ns[i][`BAR+:4] = ddr_insts[i][`DEC_BAR+:4]; + ddr_uop_ns[i][`DO_AP] = ddr_insts[i][`DEC_AP]; + ddr_uop_ns[i][`IS_BL4] = ddr_insts[i][`DEC_BL4]; + end + `PRE: begin + ddr_uop_ns[i][`IS_PRE] = `HIGH; + ddr_uop_ns[i][`PRE_ALL] = ddr_insts[i][`DEC_PRE_ALL]; + ddr_uop_ns[i][`INC_BAR] = ddr_insts[i][`DEC_INC_BAR]; + ddr_uop_ns[i][`BAR+:4] = ddr_insts[i][`DEC_BAR+:4]; + end + `ACT: begin + ddr_uop_ns[i][`IS_ACT] = `HIGH; + ddr_uop_ns[i][`INC_RAR] = ddr_insts[i][`DEC_INC_RAR]; + ddr_uop_ns[i][`INC_BAR] = ddr_insts[i][`DEC_INC_BAR]; + ddr_uop_ns[i][`RAR+:4] = ddr_insts[i][`DEC_RAR+:4]; + ddr_uop_ns[i][`BAR+:4] = ddr_insts[i][`DEC_BAR+:4]; + end + `ZQ: begin + ddr_uop_ns[i][`IS_ZQ] = `HIGH; + end + `REF: begin + ddr_uop_ns[i][`IS_REF] = `HIGH; + end + `NOP: begin + ddr_uop_ns[i][`IS_NOP] = `HIGH; + end + endcase + end + end + else if(instr[`SR_OFFSET]) begin + ddr_valid_ns = `HIGH; + case(instr[`FU_CODE_OFFSET]) + `SRE: begin + ddr_uop_ns[0][`IS_SRE] = `HIGH; + ddr_uop_ns[1][`IS_NOP] = `HIGH; + ddr_uop_ns[2][`IS_NOP] = `HIGH; + ddr_uop_ns[3][`IS_NOP] = `HIGH; + end + `SRX: begin + ddr_uop_ns[0][`IS_SRX] = `HIGH; + ddr_uop_ns[1][`IS_NOP] = `HIGH; + ddr_uop_ns[2][`IS_NOP] = `HIGH; + ddr_uop_ns[3][`IS_NOP] = `HIGH; + end + endcase + end + else begin + exe_valid_ns = `HIGH; + // Decoding a branch instruction + if(instr[`BRANCH_OFFSET]) begin + case(instr[`FU_CODE_OFFSET +: 8]) + `BL: begin + exe_uop_ns[`RS1 +: 4] = instr[`DEC_RS1 +: 4]; + exe_uop_ns[`RS2 +: 4] = instr[`DEC_RS2 +: 4]; + exe_uop_ns[`IMD +: 19] = instr[`DEC_BR_TGT_OFFSET +: 19]; + exe_uop_ns[`IS_BL] = `HIGH; + end + `BEQ: begin + exe_uop_ns[`RS1 +: 4] = instr[`DEC_RS1 +: 4]; + exe_uop_ns[`RS2 +: 4] = instr[`DEC_RS2 +: 4]; + exe_uop_ns[`IMD +: 19] = instr[`DEC_BR_TGT_OFFSET +: 19]; + exe_uop_ns[`IS_BEQ] = `HIGH; + end + `JUMP: begin + exe_uop_ns[`IMD +: 27] = instr[`DEC_JUMP_OFFSET +: 27]; + exe_uop_ns[`IS_JUMP] = `HIGH; + end + `SLEEP: begin + exe_uop_ns[`IS_SLEEP] = `HIGH; + exe_uop_ns[`IMD +: 27] = instr[`DEC_SLEEP_OFFSET +: 27]; + end + endcase + end + else if(instr[`MEM_OFFSET]) begin + case(instr[`FU_CODE_OFFSET +: 8]) + `LD: begin + exe_uop_ns[`RS1 +: 4] = instr[`DEC_RS1 +: 4]; + exe_uop_ns[`RT +: 4] = instr[`DEC_RT +: 4]; + exe_uop_ns[`IMD +: 16] = instr[`DEC_IMD1 +: 16]; + exe_uop_ns[`IS_MEM] = `HIGH; + exe_uop_ns[`IS_LD] = `HIGH; + exe_uop_ns[`HAS_IMD] = `HIGH; + end + `ST: begin + exe_uop_ns[`RS1 +: 4] = instr[`DEC_RS1 +: 4]; // The address base to write to + exe_uop_ns[`RS2 +: 4] = instr[`DEC_RT +: 4]; // The value to write + exe_uop_ns[`IMD +: 16] = instr[`DEC_IMD1 +: 16]; + exe_uop_ns[`IS_MEM] = `HIGH; + exe_uop_ns[`IS_ST] = `HIGH; + exe_uop_ns[`HAS_IMD] = `HIGH; + end + endcase + end + else if(instr[`BW_OFFSET]) begin + case(instr[`FU_CODE_OFFSET +: 8]) + `AND: begin + exe_uop_ns[`RS1 +: 4] = instr[`DEC_RS1 +: 4]; + exe_uop_ns[`RS2 +: 4] = instr[`DEC_RS2 +: 4]; + exe_uop_ns[`RT +: 4] = instr[`DEC_RT +: 4]; + exe_uop_ns[`IS_AND] = `HIGH; + end + `OR: begin + exe_uop_ns[`RS1 +: 4] = instr[`DEC_RS1 +: 4]; + exe_uop_ns[`RS2 +: 4] = instr[`DEC_RS2 +: 4]; + exe_uop_ns[`RT +: 4] = instr[`DEC_RT +: 4]; + exe_uop_ns[`IS_OR] = `HIGH; + end + `XOR: begin + exe_uop_ns[`RS1 +: 4] = instr[`DEC_RS1 +: 4]; + exe_uop_ns[`RS2 +: 4] = instr[`DEC_RS2 +: 4]; + exe_uop_ns[`RT +: 4] = instr[`DEC_RT +: 4]; + exe_uop_ns[`IS_XOR] = `HIGH; + end + endcase + end + // Decoding a normal instruction + else begin + case(instr[`FU_CODE_OFFSET +: 8]) + `ADD: begin + exe_uop_ns[`RS1 +: 4] = instr[`DEC_RS1 +: 4]; + exe_uop_ns[`RS2 +: 4] = instr[`DEC_RS2 +: 4]; + exe_uop_ns[`RT +: 4] = instr[`DEC_RT +: 4]; + exe_uop_ns[`IS_ADD] = `HIGH; + end + `ADDI: begin + exe_uop_ns[`RS1 +: 4] = instr[`DEC_RS1 +: 4]; + exe_uop_ns[`IMD +: 16] = instr[`DEC_IMD1 +: 16]; + exe_uop_ns[`RT +: 4] = instr[`DEC_RT +: 4]; + exe_uop_ns[`IS_ADD] = `HIGH; + exe_uop_ns[`HAS_IMD] = `HIGH; + end + `SUB: begin + exe_uop_ns[`RS1 +: 4] = instr[`DEC_RS1 +: 4]; + exe_uop_ns[`RS2 +: 4] = instr[`DEC_RS2 +: 4]; + exe_uop_ns[`RT +: 4] = instr[`DEC_RT +: 4]; + exe_uop_ns[`IS_SUB] = `HIGH; + end + `SUBI: begin + exe_uop_ns[`RS1 +: 4] = instr[`DEC_RS1 +: 4]; + exe_uop_ns[`IMD +: 16] = instr[`DEC_IMD1 +: 16]; + exe_uop_ns[`RT +: 4] = instr[`DEC_RT +: 4]; + exe_uop_ns[`IS_SUB] = `HIGH; + exe_uop_ns[`HAS_IMD] = `HIGH; + end + `MV: begin + exe_uop_ns[`RS1 +: 4] = instr[`DEC_RS1 +: 4]; + exe_uop_ns[`RT +: 4] = instr[`DEC_RT +: 4]; + exe_uop_ns[`IS_MOV] = `HIGH; + end + `SRC: begin + exe_uop_ns[`RS1 +: 4] = instr[`DEC_RS1 +: 4]; + exe_uop_ns[`RT +: 4] = instr[`DEC_RT +: 4]; + exe_uop_ns[`IS_SRC] = `HIGH; + end + `LI: begin + exe_uop_ns[`RT +: 4] = instr[`DEC_RT +: 4]; + exe_uop_ns[`IS_LI] = `HIGH; + exe_uop_ns[`IMD +: 16] = instr[`DEC_IMD1 +: 16]; + exe_uop_ns[`IMD2 +: 16]= instr[`DEC_IMD3 +: 16]; + exe_uop_ns[`HAS_IMD] = `HIGH; + end + `LDWD: begin + exe_uop_ns[`RS1 +: 4] = instr[`DEC_RS1 +: 4]; + // used to select word offset in wide write data register + exe_uop_ns[`RT +: 4] = instr[`DEC_WO +: 4]; + exe_uop_ns[`IS_LDWD] = `HIGH; + end + `LDPC: begin + exe_uop_ns[`RS1 +: 4] = instr[`DEC_RS1 +: 4]; + exe_uop_ns[`RT +: 4] = instr[`DEC_RT +: 4]; + exe_uop_ns[`IS_LDPC] = `HIGH; + end + endcase + end + end + end + end + + always @(posedge clk) begin + + if(rst) begin + exe_pc_r <= {`IMEM_ADDR_WIDTH{`LOW}}; + exe_uop_r <= {`EXE_UOP_WIDTH{`LOW}}; + exe_valid_r <= `LOW; + ddr_valid_r <= `LOW; + for(i = 0 ; i < 4 ; i = i + 1) + ddr_uop_r[i] <= {`DDR_UOP_WIDTH{`LOW}}; + is_started <= `LOW; + end + else begin + exe_uop_r <= exe_uop_ns; + exe_valid_r <= exe_valid_ns; + ddr_valid_r <= ddr_valid_ns; + for(i = 0 ; i < 4 ; i = i + 1) + ddr_uop_r[i] <= ddr_uop_ns[i]; + exe_pc_r <= instr_pc; + end + + if(~is_started) begin + for(i = 0 ; i < 7 ; i = i + 1) begin + ddr_stat_r[i] <= 0; + end + if(instr_pc == 1) + is_started <= 1; + end + else begin + ddr_stat_r[6] <= ddr_stat_r[6] + 1; + if(instr_valid) begin + if(instr[`DDR_OFFSET]) begin + for(i = 0 ; i < 4 ; i = i + 1) begin + case(ddr_insts[i][`DDR_CODE_OFFSET +: 3]) + `WRITE: + ddr_stat_r[0] <= ddr_stat_r[0] + 1; + `READ: + ddr_stat_r[1] <= ddr_stat_r[1] + 1; + `PRE: + ddr_stat_r[2] <= ddr_stat_r[2] + 1; + `ACT: + ddr_stat_r[3] <= ddr_stat_r[3] + 1; + `ZQ: + ddr_stat_r[4] <= ddr_stat_r[4] + 1; + `REF: + ddr_stat_r[5] <= ddr_stat_r[5] + 1; + endcase + end + end + end + end + end + + + +endmodule diff --git a/sources/hdl/verilog/diff_shift_reg.v b/sources/hdl/verilog/diff_shift_reg.v new file mode 100644 index 0000000..ae39c3a --- /dev/null +++ b/sources/hdl/verilog/diff_shift_reg.v @@ -0,0 +1,69 @@ +`timescale 1ns / 1ps +////////////////////////////////////////////////////////////////////////////////// +// Company: +// Engineer: +// +// Create Date: 12/19/2018 01:07:06 PM +// Design Name: +// Module Name: diff_shift_reg +// Project Name: +// Target Devices: +// Tool Versions: +// Description: +// +// Dependencies: +// +// Revision: +// Revision 0.01 - File Created +// Additional Comments: +// +////////////////////////////////////////////////////////////////////////////////// + + +module diff_shift_reg( + input clk, + input rst, + + input flush, + + input [15:0] in, + input in_valid, + + output[511:0] out, + output out_valid + ); + + reg [4:0] ctr_r; + reg [511:0] shift_r; + reg valid_r; + + always @(posedge clk) begin + if(rst) begin + shift_r <= 512'bX; + ctr_r <= 5'b0; + valid_r <= 1'b0; + end + else begin + if(flush) begin + valid_r <= `HIGH; + ctr_r <= 5'b0; + end + else begin + if(in_valid) begin + shift_r[16 +: 16*31] <= shift_r[0 +: 16*31]; + shift_r[0 +: 16] <= in; + ctr_r <= ctr_r + 1; + end + else begin + ctr_r <= ctr_r; + shift_r <= shift_r; + end + valid_r <= (|ctr_r) && in_valid; + end + end + end + + assign out_valid = valid_r; + assign out = shift_r; + +endmodule diff --git a/sources/hdl/verilog/dll_toggler.v b/sources/hdl/verilog/dll_toggler.v new file mode 100644 index 0000000..a6cb5bf --- /dev/null +++ b/sources/hdl/verilog/dll_toggler.v @@ -0,0 +1,206 @@ +`include "parameters.vh" + +module dll_toggler #(parameter CKE_WIDTH = 1, RANK_WIDTH = 1, DQ_WIDTH = 64, DRAM_CMD_SLOTS = 4, + DATA_BUF_ADDR_WIDTH = 5, DBUF_WIDTH = 4, DQ_BURST = 8) + +( + + input clk, + input rst, + input toggle_valid, + output reg dllt_done, + + // ---------- DDR4 Signals ---------- + output [7:0] mc_ACT_n, // DRAM ACT_n command signal for four DRAM clock cycles. + output [`ADDR_WIDTH*8-1:0] mc_ADR, // DRAM address. There are 8 bits in the fabric interface for each address bit on the DRAM bus. + output [`BANK_WIDTH*8-1:0] mc_BA, // DRAM bank address. 8 bits for each DRAM bank address. + output [`BG_WIDTH*8-1:0] mc_BG, // DRAM bank group address. + output [`CS_WIDTH*8-1:0] mc_CS_n, // DRAM CS_n + + // NOTE: CKE is transmitted within another clock domain that is 4x faster than fabric clock. + output [`CKE_WIDTH*8-1:0] mc_CKE, + output clk_sel + + ); + + wire [13:0] MR1_CONF = 14'b00001100000000; + + reg [`ADDR_WIDTH*8-1:0] ADR_ns, ADR_r; + reg [`BANK_WIDTH*8-1:0] BA_ns, BA_r; + reg [`BG_WIDTH*8-1:0] BG_ns, BG_r; + reg [`CS_WIDTH*8-1:0] CS_n_ns, CS_n_r; + reg [`ODT_WIDTH*8-1:0] ODT_ns, ODT_r; + reg [`CKE_WIDTH*8-1:0] CKE_ns, CKE_r; + + reg clk_sel_ns, clk_sel_r; + + assign mc_ACT_n = {8{`HIGH}}; + assign mc_ADR = ADR_r; + assign mc_BA = BA_r; + assign mc_BG = BG_r; + assign mc_CS_n = CS_n_r; + assign mc_CKE = CKE_r; + assign clk_sel = clk_sel_r; + + + // TODO we need to implement the following routine (TO TURN DLL OFF ONLY) + // 1 - Precharge all banks (IDLE STATE) + // 2 - Set MR1 A0 to 1 (DISABLE DLL), wait tMOD + // 3 - Enter self-refresh mode, wait until tCKSRE/tCKSRE_PAR + // 4 - Change clock frequency + // 5 - Wait at least tCKSRX (until clock signal stabilizes) + // 6 - Exit self-refresh mode, keep CKE high from now on + // if any ODT feature was enabled in self-ref. mode + // ODT signal must be LOW. + // 7 - Wait tXS and set mode registers to appropriate values + // (UG says that CL, CWL and WR may need to be updated), + // wait for another tMOD + + localparam IDLE_S = 0; + localparam IDLE_WAIT_S = 1; + localparam SET_MR_1_S = 2; + localparam WAIT_MR_1_S = 3; + localparam ENTER_SELF_REF_S = 4; + localparam WAIT_ENTER_SELF_REF_S = 5; + localparam CHANGE_CLK_FREQ_S = 6; + localparam WAIT_CHANGE_CLK_FREQ_S = 7; + localparam EXIT_SELF_REF_S = 8; + localparam WAIT_EXIT_SELF_REF_S = 9; + + localparam T_PRECHARGE = 5; // in terms of MC cycles (which is 4x less frequent than the DDR4) + localparam T_MOD = 24; // Max(24CK,15ns) + localparam T_CKSRE = 300; // Max(5CK,10ns) + localparam T_CKSRX = 300; // Max(5CK, 10ns) + additional room for clock to stabilize; + localparam T_XS = 1000; + + + reg[3:0] state_r, state_ns; + + reg[9:0] wait_r, wait_ns; + + integer adr_bit_i; + + always @* begin + // Set bank and bank group signals + dllt_done = `LOW; + ADR_ns = {`ADDR_WIDTH*8{`HIGH}}; + BG_ns = {`BG_WIDTH*8{`LOW}}; + BA_ns = {`BANK_WIDTH*8{`LOW}}; + CS_n_ns = {`CS_WIDTH*8{`HIGH}}; // by default we don't issue any commands + CKE_ns = CKE_r; // register this signal because it needs to be LOW during self-ref. + wait_ns = wait_r; + state_ns = state_r; + clk_sel_ns = clk_sel_r; + case (state_r) + IDLE_S: begin + if(toggle_valid) begin + CS_n_ns[1:0] = {2*`CS_WIDTH{`LOW}}; + ADR_ns[`ADDR_WIDTH*8-3*8 +: 2] = {2{`LOW}}; // WE + ADR_ns[`ADDR_WIDTH*8-2*8 +: 2] = {2{`HIGH}}; // ~CAS + ADR_ns[`ADDR_WIDTH*8-8 +: 2] = {2{`LOW}}; // RAS + ADR_ns[10*8 +: 2] = {2{`HIGH}}; // Pre ALL + wait_ns = T_PRECHARGE; + state_ns = IDLE_WAIT_S; + end + end + IDLE_WAIT_S: begin + if(wait_r > 0) + wait_ns = wait_r - 1'b1; + else + state_ns = SET_MR_1_S; + end + SET_MR_1_S: begin + CS_n_ns[1:0] = {2*`CS_WIDTH{`LOW}}; + ADR_ns[`ADDR_WIDTH*8-3*8 +: 2] = {2{`LOW}}; // WE + ADR_ns[`ADDR_WIDTH*8-2*8 +: 2] = {2{`LOW}}; // CAS + ADR_ns[`ADDR_WIDTH*8-8 +: 2] = {2{`LOW}}; // RAS + for(adr_bit_i = 0 ; adr_bit_i < 14 ; adr_bit_i = adr_bit_i + 1) begin + ADR_ns[adr_bit_i*8 +: 2] = + {2{MR1_CONF[adr_bit_i]}}; + end + // Bank + Bank group bits indicate which register this MRS is writing to. + BA_ns[0 +: 2] = {2{`HIGH}}; // Select MR1 + ADR_ns[0 +: 2] = {2{`LOW}}; // Set A0 to 0 + state_ns = WAIT_MR_1_S; + wait_ns = T_MOD; + end + WAIT_MR_1_S: begin + if(wait_r > 0) + wait_ns = wait_r - 1'b1; + else + state_ns = ENTER_SELF_REF_S; + end + ENTER_SELF_REF_S: begin + CKE_ns = `LOW; + CS_n_ns[1:0] = {2*`CS_WIDTH{`LOW}}; + ADR_ns[`ADDR_WIDTH*8-3*8 +: 2] = {2{`HIGH}}; // ~WE + ADR_ns[`ADDR_WIDTH*8-2*8 +: 2] = {2{`LOW}}; // CAS + ADR_ns[`ADDR_WIDTH*8-8 +: 2] = {2{`LOW}}; // RAS + state_ns = WAIT_ENTER_SELF_REF_S; + wait_ns = T_CKSRE; + end + WAIT_ENTER_SELF_REF_S: begin + if(wait_r > 0) + wait_ns = wait_r - 1'b1; + else begin + state_ns = CHANGE_CLK_FREQ_S; + end + end + CHANGE_CLK_FREQ_S: begin + clk_sel_ns = ~clk_sel_r; + wait_ns = T_CKSRX; + state_ns = WAIT_CHANGE_CLK_FREQ_S; + end + WAIT_CHANGE_CLK_FREQ_S: begin + if(wait_r > 0) + wait_ns = wait_r - 1'b1; + else begin + state_ns = EXIT_SELF_REF_S; + end + end + EXIT_SELF_REF_S: begin + CKE_ns = {`CKE_WIDTH*8{`HIGH}}; + CS_n_ns[7:0] = {8*`CS_WIDTH{`HIGH}}; + ADR_ns[`ADDR_WIDTH*8-3*8 +: 2] = {2{`HIGH}}; // ~WE + ADR_ns[`ADDR_WIDTH*8-2*8 +: 2] = {2{`HIGH}}; // CAS + ADR_ns[`ADDR_WIDTH*8-8 +: 2] = {2{`HIGH}}; // RAS + state_ns = WAIT_EXIT_SELF_REF_S; + wait_ns = T_XS; + end + WAIT_EXIT_SELF_REF_S: begin + CS_n_ns[7:0] = {8*`CS_WIDTH{`HIGH}}; + if(wait_r > 0) + wait_ns = wait_r - 1'b1; + else begin + state_ns = IDLE_S; + dllt_done = `HIGH; + end + end + endcase + + end + + always @(posedge clk) begin + if(rst) begin + state_r <= IDLE_S; + wait_r <= `LOW; + clk_sel_r <= `LOW; + CKE_r <= {`CKE_WIDTH*8{`HIGH}}; + ADR_r = {`ADDR_WIDTH*8{`LOW}}; + BG_r = {`BG_WIDTH*8{`LOW}}; + BA_r = {`BANK_WIDTH*8{`LOW}}; + CS_n_r = {`CS_WIDTH*8{`HIGH}}; + end + else begin + state_r <= state_ns; + wait_r <= wait_ns; + clk_sel_r <= clk_sel_ns; + CKE_r <= CKE_ns; + ADR_r <= ADR_ns; + BA_r <= BA_ns; + BG_r <= BG_ns; + CS_n_r <= CS_n_ns; + end + end + +endmodule diff --git a/sources/hdl/verilog/exe_pipeline.v b/sources/hdl/verilog/exe_pipeline.v new file mode 100644 index 0000000..e2111a3 --- /dev/null +++ b/sources/hdl/verilog/exe_pipeline.v @@ -0,0 +1,191 @@ +`include "parameters.vh" +`include "encoding.vh" + +module exe_pipeline( + // common signals + input clk, + input rst, + + // exe_pipeline <-> execution stage if + input exe_valid, + input [`EXE_UOP_WIDTH-1:0] exe_uop, + input [`IMEM_ADDR_WIDTH-1:0] exe_pc, + + // branch unit if + output br_resolve, + output [`IMEM_ADDR_WIDTH-1:0] br_target, + + // exe_pipeline <-> register file if + output wide_wen, + output [31:0] rf_wdata, + output rf_wen, + output [7:0] rf_raddr, + output [3:0] rf_waddr, + input [2*32-1:0] rf_rdata, + input [32*7-1:0] ddr_stat, + + // exe_pipeline <-> scratchpad + output mem_wen, + output mem_ren, + output [9:0] mem_addr, + output [31:0] mem_wdata, + input [31:0] mem_rdata + ); + + // calculated at stage one, registers + reg s2_valid; + reg [`EXE_UOP_WIDTH-1:0] s2_uop; + reg [`IMEM_ADDR_WIDTH-1:0] s2_pc; + reg [3:0] s2_rs1; + reg [3:0] s2_rs2; + reg [3:0] s2_rt; + reg s2_wen; + reg s2_mem_ren; + reg s2_mem_wen; + reg s3_wen; // for loads + reg [3:0] s3_rt; // for loads + reg s2_wide_wen; + reg [31:0] s2_imd_r, s2_imd_ns; + + // delayed branch resolution signals + // calculated at stage two, registers + reg s3_br_resolve; + reg [`IMEM_ADDR_WIDTH-1:0] s3_br_target; + + // combinational elements + reg [31:0] s2_wdata; + wire [31:0] s2_rs1_data, s2_rs2_data; + reg [31:0] s2_mem_wdata; + reg [`IMEM_ADDR_WIDTH-1:0] fetch_pc; + + assign wide_wen = s2_wide_wen; + assign rf_wdata = s3_wen ? mem_rdata : s2_wdata; + assign rf_wen = s2_wen || s3_wen; + assign rf_raddr[0+:4] = s2_rs1; + assign rf_raddr[4+:4] = s2_rs2; + assign rf_waddr = s3_wen ? s3_rt : s2_rt; + + assign s2_rs1_data = rf_rdata[0+:32]; + assign s2_rs2_data = rf_rdata[32+:32]; + assign br_resolve = s3_br_resolve; + assign br_target = s3_br_target; + + assign mem_addr = s2_rs1_data + s2_imd_r; + assign mem_wen = s2_mem_wen; + assign mem_ren = s2_mem_ren; + assign mem_wdata = s2_mem_wdata; + + always @* begin + // stage one, decode immediate value + s2_mem_wen = `LOW; + s2_imd_ns = s2_imd_r; + fetch_pc = {`IMEM_ADDR_WIDTH{1'bX}}; + s2_wdata = 32'bX; + s2_mem_wen = `LOW; + s2_mem_ren = `LOW; + if(exe_uop[`HAS_IMD]) begin + s2_imd_ns[15:0] = exe_uop[`IMD +: 16]; + s2_imd_ns[31:16] = {16{`LOW}}; + end + if(exe_uop[`IS_LI]) begin + s2_imd_ns[31:16] = exe_uop[`IMD2 +: 16]; + end + if(exe_uop[`IS_BL] | exe_uop[`IS_BEQ]) begin + s2_imd_ns[31:0] = exe_uop[`IMD +: 19]; + end + if(exe_uop[`IS_JUMP]) begin + s2_imd_ns[31:0] = exe_uop[`IMD +: 27]; + end + // stage two, actual computation + if(s2_uop[`IS_ADD]) begin + if(s2_uop[`HAS_IMD]) + s2_wdata = s2_rs1_data + s2_imd_r; + else + s2_wdata = s2_rs1_data + s2_rs2_data; + end + if(s2_uop[`IS_SUB]) begin + if(s2_uop[`HAS_IMD]) + s2_wdata = s2_rs1_data - s2_imd_r; + else + s2_wdata = s2_rs1_data - s2_rs2_data; + end + if(s2_uop[`IS_MOV] || s2_uop[`IS_LDWD]) begin + s2_wdata = s2_rs1_data; + end + if(s2_uop[`IS_LI]) begin + s2_wdata = s2_imd_r; + end + if(s2_uop[`IS_LDPC]) begin + s2_wdata = ddr_stat[s2_rs1*32 +: 32]; + end + if(s2_uop[`IS_SRC]) begin + s2_wdata[30:0] = s2_rs1_data[31:1]; + s2_wdata[31] = s2_rs1_data[0]; + end + // Bitwise ops + if(s2_uop[`IS_AND]) begin + s2_wdata = s2_rs1_data & s2_rs2_data; + end + if(s2_uop[`IS_OR]) begin + s2_wdata = s2_rs1_data | s2_rs2_data; + end + if(s2_uop[`IS_XOR]) begin + s2_wdata = s2_rs1_data ^ s2_rs2_data; + end + // mem unit + if(s2_uop[`IS_LD]) begin + s2_mem_ren = `HIGH; + end + if(s2_uop[`IS_ST]) begin + s2_mem_wdata = s2_rs2_data; + s2_mem_wen = `HIGH; + end + + // branch unit + if(s2_uop[`IS_BL]) begin + if(s2_rs1_data < s2_rs2_data) + fetch_pc = s2_imd_r; + else // not taken + fetch_pc = s2_pc + 1; + end + if(s2_uop[`IS_BEQ]) begin + if(s2_rs1_data == s2_rs2_data) + fetch_pc = s2_imd_r; + else // not taken + fetch_pc = s2_pc + 1; + end + if(s2_uop[`IS_JUMP]) begin + fetch_pc = s2_imd_r; + end + if(s2_uop[`IS_SLEEP]) begin + // not implemented + // frontent/fetch handles api sleeps + end + + end + + always @(posedge clk) begin + // stage one, further decode + s2_valid <= exe_valid; + s2_uop <= exe_uop; + s2_pc <= exe_pc; + s2_rs1 <= exe_uop[`RS1 +: 4]; + s2_rs2 <= exe_uop[`RS2 +: 4]; + s2_rt <= exe_uop[`RT +: 4]; + s2_imd_r <= s2_imd_ns; + s2_wen <= exe_uop[`IS_ADD] || exe_uop[`IS_SUB] || + exe_uop[`IS_MOV] || exe_uop[`IS_LI] || + exe_uop[`IS_LDWD] || exe_uop[`IS_SRC] || + exe_uop[`IS_AND] || exe_uop[`IS_OR] || + exe_uop[`IS_XOR] || exe_uop[`IS_LDPC]; + s2_wide_wen <= exe_uop[`IS_LDWD]; + // stage two, delayed branch signals + s3_br_resolve <= s2_valid && (s2_uop[`IS_BL] || + s2_uop[`IS_BEQ] || + s2_uop[`IS_JUMP]); + s3_br_target <= fetch_pc; + s3_wen <= s2_uop[`IS_LD]; // write path for loads + s3_rt <= s2_uop[`RT +: 4]; + end + +endmodule diff --git a/sources/hdl/verilog/execute_stage.v b/sources/hdl/verilog/execute_stage.v new file mode 100644 index 0000000..0b690f8 --- /dev/null +++ b/sources/hdl/verilog/execute_stage.v @@ -0,0 +1,194 @@ +`include "parameters.vh" + +module execute_stage( + + // common signals + input clk, + input rst, + + // decode stage <-> execute stage if + input ddr_valid, + input exe_valid, + input [`DDR_UOP_WIDTH*4-1:0] ddr_uop, + input [`EXE_UOP_WIDTH-1:0] exe_uop, + input [`IMEM_ADDR_WIDTH-1:0] exe_pc, + + // branch unit <-> fetch stage if + output br_resolve, + output [`IMEM_ADDR_WIDTH-1:0] br_target, + + // execute <-> outer DDRX IP interface + output [3:0] ddr_write, + output [3:0] ddr_read, + output [3:0] ddr_pre, + output [3:0] ddr_act, + output [3:0] ddr_ref, + output [3:0] ddr_sre, + output [3:0] ddr_srx, + output [3:0] ddr_zq, + output [3:0] ddr_nop, + output [3:0] ddr_ap, + output [3:0] ddr_pall, + output [3:0] ddr_half_bl, + output [4*`BG_WIDTH-1:0] ddr_bg, + output [4*`BANK_WIDTH-1:0] ddr_bank, + output [4*`COL_WIDTH-1:0] ddr_col, + output [4*`ROW_WIDTH-1:0] ddr_row, + output [511:0] ddr_wdata, + + // execute <-> reg file if + output rf_wide_wen, + output [3:0] rf_wide_offset, + input [32*8-1:0] rf_rdata, + output [32*8-1:0] rf_wdata, + output [7:0] rf_wen, + output [4*8-1:0] rf_raddr, + output [4*8-1:0] rf_waddr, + input [`COL_WIDTH-1:0] casr, + input [`BANK_WIDTH+`BG_WIDTH-1:0] basr, + input [`ROW_WIDTH-1:0] rasr, + input [511:0] wide_reg, + + // update stride registers, wen is OH (3'b001 = update rasr) + output [31:0] srf_value, + output [2:0] srf_wen, + input [32*7-1:0] ddr_stat + ); + + reg s2_exe; // exe_pipeline has regfile ports + + // ddr operations can optionally update + // registers. TODO when do we read the + // stride values? + wire [7:0] rf_wen_ddr; + wire [4*8-1:0] rf_waddr_ddr; + wire [32*8-1:0] rf_wdata_ddr; + wire [4*8-1:0] rf_raddr_ddr; + + ddr_pipeline dp( + .clk(clk), + .rst(rst), + + // execute <-> ddr_pipe if + .ddr_valid(ddr_valid), + .ddr_uop(ddr_uop), + + // ddr_pipeline <-> outer DDRX IP interface + .ddr_write(ddr_write), + .ddr_read(ddr_read), + .ddr_pre(ddr_pre), + .ddr_act(ddr_act), + .ddr_ref(ddr_ref), + .ddr_sre(ddr_sre), + .ddr_srx(ddr_srx), + .ddr_zq(ddr_zq), + .ddr_nop(ddr_nop), + .ddr_ap(ddr_ap), + .ddr_pall(ddr_pall), + .ddr_half_bl(ddr_half_bl), + .ddr_bg(ddr_bg), + .ddr_bank(ddr_bank), + .ddr_col(ddr_col), + .ddr_row(ddr_row), + .ddr_wdata(ddr_wdata), + // ddr_pipeline <-> regfile interface + .update_en(rf_wen_ddr), + .update_ids(rf_waddr_ddr), + .update_vals(rf_wdata_ddr), + .casr(casr), + .basr(basr), + .rasr(rasr), + .reg_ids(rf_raddr_ddr), // registers we need to read + .reg_vals(rf_rdata), // register values + .wide_reg(wide_reg) // write data register + ); + + + wire[31:0] rf_wdata_exe; + wire[7:0] rf_raddr_exe; + wire[3:0] rf_waddr_exe; + wire rf_wen_exe; + wire rf_wide_wen_exe; + + wire mem_wen; + wire mem_ren; + wire[9:0] mem_addr; + wire[31:0] mem_wdata; + wire[31:0] mem_rdata; + + exe_pipeline ep( + .clk(clk), + .rst(rst), + + // exe_pipeline <-> execute stage if + .exe_valid(exe_valid), + .exe_uop(exe_uop), + .exe_pc(exe_pc), + + // branch unit <-> fetch stage if + .br_resolve(br_resolve), + .br_target(br_target), + + // exe_pipeline <-> regfile if + .wide_wen(rf_wide_wen_exe), + .rf_wdata(rf_wdata_exe), + .rf_wen(rf_wen_exe), + .rf_raddr(rf_raddr_exe), + .rf_rdata(rf_rdata[0 +: 32*2]), + .rf_waddr(rf_waddr_exe), + + // exe_pipeline <-> data_mem + .mem_wen(mem_wen), + .mem_ren(mem_ren), + .mem_addr(mem_addr), + .mem_wdata(mem_wdata), + .mem_rdata(mem_rdata), + .ddr_stat(ddr_stat) + + ); +`ifdef XILINX_SIMULATOR + reg_mem data_mem( + .addr(mem_addr), + .clk(clk), + .din(mem_wdata), + .dout(mem_rdata), + .en(mem_wen || mem_ren), + .we(mem_wen) + ); +`else + scratchpad data_mem( + .addra(mem_addr), + .clka(clk), + .dina(mem_wdata), + .douta(mem_rdata), + .ena(mem_wen || mem_ren), + .wea(mem_wen) + ); +`endif + + wire exe_target_srf = rf_waddr_exe == 4'b0000 || + rf_waddr_exe == 4'b0001 || + rf_waddr_exe == 4'b0010; + + assign rf_wide_wen = rf_wide_wen_exe; + assign rf_wide_offset = rf_waddr_exe; + assign rf_wdata[32 +: 7*32] = rf_wdata_ddr[32 +: 7*32]; + assign rf_wdata[0 +: 32] = rf_wen_exe ? rf_wdata_exe : rf_wdata_ddr[0 +: 32]; + assign rf_wen = (rf_wen_exe & ~rf_wide_wen_exe & ~exe_target_srf) | rf_wen_ddr; + assign rf_raddr[8 +: 6*4] = rf_raddr_ddr[8 +: 6*4]; + assign rf_raddr[0 +: 2*4] = s2_exe ? rf_raddr_exe : rf_raddr_ddr[0 +: 2*8]; + assign rf_waddr[4 +: 7*4] = rf_waddr_ddr[4 +: 7*4]; + assign rf_waddr[0 +: 4] = rf_wen_exe ? rf_waddr_exe : rf_waddr_ddr[0 +: 4]; + + // First 3 register ids implicitly target stride registers + assign srf_wen[0] = rf_wen_exe & ~rf_wide_wen_exe & (rf_waddr_exe == 4'b0000); + assign srf_wen[1] = rf_wen_exe & ~rf_wide_wen_exe & (rf_waddr_exe == 4'b0001); + assign srf_wen[2] = rf_wen_exe & ~rf_wide_wen_exe & (rf_waddr_exe == 4'b0010); + + assign srf_value = rf_wdata_exe; + + always @(posedge clk) begin + s2_exe <= exe_valid; + end + +endmodule diff --git a/sources/hdl/verilog/fetch_stage.v b/sources/hdl/verilog/fetch_stage.v new file mode 100644 index 0000000..bd72fe5 --- /dev/null +++ b/sources/hdl/verilog/fetch_stage.v @@ -0,0 +1,147 @@ +`include "parameters.vh" + +module fetch_stage( + // common signals + input clk, + input rst, + + // other control signals + output softmc_end, + output [11:0] read_size, + output reg read_seq_incoming, + input [11:0] buffer_space, + + // branch unit <-> fetch stage interface + input br_resolve, + input [`IMEM_ADDR_WIDTH-1:0] br_target, + + // fetch stage <-> frontend interface + output [`IMEM_ADDR_WIDTH-1:0] addr_out, + output valid_out, + input [`INSTR_WIDTH-1:0] data_in, + input valid_in, + input [`IMEM_ADDR_WIDTH-1:0] addr_in, + input ready_out, // frontend is ready for a valid request + + // fetch stage <-> decode stage interface + output [`INSTR_WIDTH-1:0] instr, + output [`IMEM_ADDR_WIDTH-1:0] instr_pc, + output instr_valid + ); + + wire inst_is_br, is_end, is_ddr_start, need_flush, is_sleep; + + reg [31:0] sleep_ctr_r, sleep_ctr_ns; + + localparam WAIT_RESOLVE_S = 0; + localparam FETCH_NEXT_LINE_S = 1; + localparam WAIT_BUFFER_SPACE_S = 2; + localparam WAIT_SLEEP_S = 3; + + reg [2:0] state_r, state_ns; + + // kind of confusing but these PCs map to + // an instruction instead of to a byte. + // i.e. each PC addresses an instruction. + reg [`IMEM_ADDR_WIDTH-1:0] pc_r, pc_ns; + + // register outputs, decode will receive + // stuff we've received with one cycle latency + // i.e. marks the end of fetch_stage cycle + reg [`IMEM_ADDR_WIDTH-1:0] instr_pc_r, instr_pc_ns; + reg [`INSTR_WIDTH-1:0] instr_r, instr_ns; + reg instr_valid_r, instr_valid_ns; + + pre_decode pdec( + .buffer_space(buffer_space), + .read_size(read_size), + .instruction(state_r == WAIT_BUFFER_SPACE_S ? instr_r : data_in), + .is_branch(inst_is_br), + .is_end(is_end), + .is_ddr_start(is_ddr_start), + .need_flush(need_flush), + .is_sleep(is_sleep) + ); + + assign instr = instr_r; + assign instr_valid = instr_valid_r; + assign instr_pc = instr_pc_r; + + // request instr @ pc from frontend + assign valid_out = ready_out && (state_r == FETCH_NEXT_LINE_S) && ~need_flush && ~(valid_in && is_sleep); + assign addr_out = pc_r; + + assign softmc_end = is_end && valid_in && (state_r == FETCH_NEXT_LINE_S); + + always @* begin + sleep_ctr_ns = sleep_ctr_r; + state_ns = state_r; + pc_ns = pc_r; + instr_ns = instr_r; + instr_pc_ns = instr_pc_r; + instr_valid_ns = ~is_end && valid_in && (state_r == FETCH_NEXT_LINE_S) + && ~is_ddr_start; + read_seq_incoming = `LOW; + case(state_r) + WAIT_RESOLVE_S: begin + if(br_resolve) begin + state_ns = FETCH_NEXT_LINE_S; + pc_ns = br_target; + end + end + FETCH_NEXT_LINE_S: begin + if(ready_out && valid_out) + pc_ns = pc_r + 1; + if(valid_in) begin + instr_ns = data_in; + instr_pc_ns = addr_in; + if(is_sleep) begin + sleep_ctr_ns = data_in[31:0]; + state_ns = WAIT_SLEEP_S; + end + if(is_ddr_start && ~need_flush && (|data_in[9:0])) + read_seq_incoming = `HIGH; + if(inst_is_br && ~is_sleep) // we don't have the ability to perform well + state_ns = WAIT_RESOLVE_S; + else if(is_end) + pc_ns = {`IMEM_ADDR_WIDTH{`LOW}}; + else if(need_flush) begin + state_ns = WAIT_BUFFER_SPACE_S; + pc_ns = addr_in; // register the info packet + // as the next instruction to fetch + end + end + end + WAIT_BUFFER_SPACE_S: begin + if(~need_flush) begin + state_ns = FETCH_NEXT_LINE_S; + end + end + WAIT_SLEEP_S: begin + sleep_ctr_ns = sleep_ctr_r - 1; + if(sleep_ctr_r == 32'b1) + state_ns = FETCH_NEXT_LINE_S; + end + endcase + end + + always @(posedge clk) begin + if (rst) begin + pc_r <= {`IMEM_ADDR_WIDTH{`LOW}}; + state_r <= FETCH_NEXT_LINE_S; + instr_valid_r <= `LOW; + instr_r <= {`INSTR_WIDTH{`LOW}}; + instr_pc_r <= {`IMEM_ADDR_WIDTH{`LOW}}; + sleep_ctr_r <= {32{`LOW}}; + end + else begin + state_r <= state_ns; + pc_r <= pc_ns; + instr_r <= instr_ns; + instr_pc_r <= instr_pc_ns; + instr_valid_r <= instr_valid_ns; + sleep_ctr_r <= sleep_ctr_ns; + end + end + +endmodule diff --git a/sources/hdl/verilog/frontend.v b/sources/hdl/verilog/frontend.v new file mode 100644 index 0000000..9225bd9 --- /dev/null +++ b/sources/hdl/verilog/frontend.v @@ -0,0 +1,237 @@ +`include "parameters.vh" +/* + * This module is responsible for the interface + * between XDMA IP and the fetch stage. + * This module encapsulates a X KiB BRAM which + * is used as an instruction memory. + */ +module frontend#(parameter SIM_MEM = "false")( + // common signals + input clk, + input rst, + + // other control signals + input softmc_fin, + output user_rst, + input init_calib_complete, + output reg rbe_switch_mode, + output reg dllt_begin, + output frontend_ready, + + // frontend <-> fetch stage interface + input [`IMEM_ADDR_WIDTH-1:0] addr_in, + input valid_in, + output [`INSTR_WIDTH-1:0] data_out, + output valid_out, + output [`IMEM_ADDR_WIDTH-1:0] addr_out, + output ready_in, + + // frontend <-> xdma interface + input [`XDMA_AXI_DATA_WIDTH-1:0] h2c_tdata_0, + input h2c_tlast_0, + input h2c_tvalid_0, + output h2c_tready_0, + input [`XDMA_AXI_DATA_WIDTH/8-1:0] h2c_tkeep_0, + + // maintenance signals + output per_rd_init, + output per_zq_init, + output per_ref_init + ); + + reg[31:0] delay_fin; + + always @(posedge clk) begin + if(rst || user_rst) + delay_fin <= 32'b0; + else + delay_fin[1+:31] <= delay_fin[0+:31]; + delay_fin[0] <= softmc_fin; + end + + assign frontend_ready = delay_fin[31]; + + wire imem_wr_en, imem_rd_en; + wire [`IMEM_ADDR_WIDTH-1:0] imem_addr; + wire [`INSTR_WIDTH-1:0] imem_wr_data, imem_rd_data; + + generate + if(SIM_MEM == "true") begin + instr_blk_mem_sim imem( + .addra(imem_addr), + .clka(clk), + .dina(imem_wr_data), + .douta(imem_rd_data), + .ena(imem_rd_en || imem_wr_en), + .wea(imem_wr_en) + ); + end + else begin + instr_blk_mem imem( + .addra(imem_addr), + .clka(clk), + .dina(imem_wr_data), + .douta(imem_rd_data), + .ena(imem_rd_en || imem_wr_en), + .wea(imem_wr_en) + ); + end + endgenerate + + wire [`INSTR_WIDTH-1:0] maint_inst; + wire maint_valid; + wire [`IMEM_ADDR_WIDTH-1:0] maint_addr; + wire maint_req; + reg maint_ack; + wire maint_process; + wire program_process; + reg aref_en; + reg aref_en_valid; + + maintenance_controller maint_ctrl + ( + .clk(clk), + .rst(rst | user_rst), + + .init_calib_complete(init_calib_complete), + .softmc_fin(softmc_fin), + + .aref_en(aref_en), + .aref_en_valid(aref_en_valid), + .maint_req(maint_req), + .maint_ack(maint_ack), + .per_rd_init(per_rd_init), + .per_zq_init(per_zq_init), + .per_ref_init(per_ref_init), + .maint_process(maint_process), + .program_process(program_process), + + .in_addr(addr_in), + .in_valid(valid_in), + + .out_data(maint_inst), + .out_valid(maint_valid), + .out_addr(maint_addr) + ); + + localparam IDLE_S = 2'd0; + localparam INIT_MEM_S = 2'd1; + localparam EXECUTE_S = 2'd2; + + reg [1:0] state_r, state_ns; + + reg [4:0] rst_ctr_ns, rst_ctr_r; + reg [`IMEM_ADDR_WIDTH-1:0] xfer_ctr_r, xfer_ctr_ns; + reg [`IMEM_RD_LATENCY-1:0] valid_out_sr; + reg [(`IMEM_RD_LATENCY * `IMEM_ADDR_WIDTH)-1:0] addr_out_sr; + + assign user_rst = (|rst_ctr_r); + + // imem <-> xdma interface + // TODO do we need tkeep? + assign h2c_tready_0 = state_r == INIT_MEM_S; + assign imem_wr_en = h2c_tvalid_0 && (state_r == INIT_MEM_S); + assign imem_wr_data = h2c_tdata_0[`INSTR_WIDTH-1:0]; + assign imem_addr = state_r == INIT_MEM_S ? xfer_ctr_r : addr_in; + // imem <-> pipeline interface + assign imem_rd_en = valid_in && (program_process); + assign data_out = program_process ? imem_rd_data : maint_inst; + assign valid_out = program_process ? valid_out_sr[0] : maint_valid; + assign addr_out = program_process ? addr_out_sr[`IMEM_ADDR_WIDTH-1:0] : maint_addr; + + generate + if(SIM_MEM=="false") + assign ready_in = state_r == EXECUTE_S; + else + assign ready_in = state_r == EXECUTE_S && ~rst; + endgenerate + assign program_process = (state_r == EXECUTE_S) && ~maint_process; + + always @* begin + aref_en_valid = `LOW; + aref_en = `LOW; + state_ns = state_r; + xfer_ctr_ns = xfer_ctr_r; + rst_ctr_ns = {5{`LOW}}; + maint_ack = `LOW; + rbe_switch_mode = `LOW; + dllt_begin = `LOW; + case (state_r) + IDLE_S: begin + if(~((|delay_fin) || softmc_fin)) begin + if(h2c_tvalid_0) + state_ns = INIT_MEM_S; + else begin + if(maint_req) begin + maint_ack = `HIGH; + state_ns = EXECUTE_S; + end + end + end + end + INIT_MEM_S: begin + if(h2c_tvalid_0) begin + if(h2c_tdata_0[`INSTR_WIDTH]) //indicates a reset + rst_ctr_ns = {5{1'b1}}; + else if(h2c_tdata_0[`INSTR_WIDTH+1]) // indicate switch between readback modes + rbe_switch_mode = `HIGH; + else if(h2c_tdata_0[`INSTR_WIDTH+2]) // indicate dll toggle off WIP + dllt_begin = `HIGH; + else if(h2c_tdata_0[`INSTR_WIDTH+3]) begin // enable-disable autoref + aref_en_valid = `HIGH; + aref_en = h2c_tdata_0[0]; + state_ns = IDLE_S; + end + else begin + xfer_ctr_ns = xfer_ctr_r + 1; + if(h2c_tlast_0) begin + state_ns = EXECUTE_S; + xfer_ctr_ns = {`IMEM_ADDR_WIDTH{`LOW}}; + end + end + end + end + EXECUTE_S: begin + if(h2c_tvalid_0) begin + if(h2c_tdata_0[`INSTR_WIDTH]) //indicates a reset + rst_ctr_ns = {5{1'b1}}; + end + if(softmc_fin) + state_ns = IDLE_S; + end + endcase + + end + + always @(posedge clk) begin + if(rst || (|rst_ctr_r)) begin + if(SIM_MEM == "false") + state_r <= IDLE_S; + else + state_r <= EXECUTE_S; + xfer_ctr_r <= {`IMEM_ADDR_WIDTH{`LOW}}; + valid_out_sr <= {`IMEM_RD_LATENCY{`LOW}}; + addr_out_sr <= 0; + if(rst_ctr_r > 0) + rst_ctr_r <= rst_ctr_r - 1; + else + rst_ctr_r <= 0; + end + else begin + state_r <= state_ns; + xfer_ctr_r <= xfer_ctr_ns; + rst_ctr_r <= rst_ctr_ns; + // compute when we should assert valid data to + // fetch stage. + valid_out_sr[`IMEM_RD_LATENCY-1] <= valid_in && (state_r == EXECUTE_S); + addr_out_sr[`IMEM_RD_LATENCY*`IMEM_ADDR_WIDTH-1 : + (`IMEM_RD_LATENCY-1)*`IMEM_ADDR_WIDTH] <= addr_in; + `ifdef IMEM_SR + valid_out_sr[`IMEM_RD_LATENCY-1:0] <= valid_out_sr >> 1; + addr_out_sr[(`IMEM_RD_LATENCY-1)*`IMEM_ADDR_WIDTH-1:0] + <= addr_out_sr >> `IMEM_ADDR_WIDTH; + `endif + end + end +endmodule + diff --git a/sources/hdl/verilog/maintenance_controller.v b/sources/hdl/verilog/maintenance_controller.v new file mode 100644 index 0000000..9354d17 --- /dev/null +++ b/sources/hdl/verilog/maintenance_controller.v @@ -0,0 +1,265 @@ +`include "parameters.vh" + +module maintenance_controller#(parameter tCK = 1500)( + + input clk, + input rst, + + input init_calib_complete, + input softmc_fin, + + input aref_en, + input aref_en_valid, + + input maint_ack, + output maint_req, + output per_rd_init, + output per_zq_init, + output per_ref_init, + output maint_process, + input program_process, + + input [`IMEM_ADDR_WIDTH-1:0] in_addr, + input in_valid, + + output [`INSTR_WIDTH-1:0] out_data, + output reg out_valid, + output [`IMEM_ADDR_WIDTH-1:0] out_addr + ); + + wire zq_ack, per_rd_ack, ref_ack; + reg zq_request, zq_process; + reg per_rd_request, per_rd_process; + reg pr_ref_request, pr_ref_process; + + wire [`INSTR_WIDTH-1 : 0] pr_read_out, pr_zq_out, pr_ref_out; + + assign maint_req = per_rd_request | zq_request | pr_ref_request; + assign out_addr = {`IMEM_ADDR_WIDTH{`HIGH}}; + assign out_data = zq_process ? pr_zq_out : per_rd_process ? pr_read_out : pr_ref_out; + assign per_ref_init = maint_ack && pr_ref_request && ~per_rd_request && ~zq_request; + assign per_rd_init = maint_ack && per_rd_request && ~zq_request; + assign per_zq_init = maint_ack && zq_request; + assign maint_process = zq_process || per_rd_process || pr_ref_process; + + assign zq_ack = softmc_fin && zq_process; + assign per_rd_ack = softmc_fin && per_rd_process; + assign ref_ack = softmc_fin && pr_ref_process; + + pr_read_mem prm + ( + .addra(in_addr[3:0]), + .clka(clk), + .douta(pr_read_out), + .dina(64'bX), + .wea(`LOW), + .ena(in_valid && per_rd_process) + ); + + zq_calib_mem pzm + ( + .addra(in_addr[5:0]), + .clka(clk), + .douta(pr_zq_out), + .dina(64'bX), + .wea(`LOW), + .ena(in_valid && zq_process) + ); + + pr_ref_mem prefm + ( + .addra(in_addr[6:0]), + .clka(clk), + .douta(pr_ref_out), + .dina(64'bX), + .wea(`LOW), + .ena(in_valid && pr_ref_process) + ); + + always @(posedge clk) begin + if(rst) begin + zq_process <= `LOW; + per_rd_process <= `LOW; + out_valid <= `LOW; + pr_ref_process <= `LOW; + end + else begin + if(softmc_fin) begin + if(zq_process) + zq_process <= `LOW; + if(per_rd_process) + per_rd_process <= `LOW; + if(pr_ref_process) + pr_ref_process <= `LOW; + out_valid <= `LOW; + end + else if(maint_ack) begin + if(zq_request) + zq_process <= `HIGH; + else if(per_rd_request) + per_rd_process <= `HIGH; + else if(pr_ref_request) + pr_ref_process <= `HIGH; + out_valid <= `LOW; + end + else begin + zq_process <= zq_process; + per_rd_process <= per_rd_process; + pr_ref_process <= pr_ref_process; + if(maint_process) + out_valid <= in_valid; + else + out_valid <= `LOW; + end + end + end + + // Maintenance control logic + // Set request bits according to timers + + localparam MAINT_PRESCALER_PERIOD = 500_000; // .5 us + + function integer clogb2 (input integer size); // ceiling logb2 + begin + size = size - 1; + for (clogb2=1; size>1; clogb2=clogb2+1) + size = size >> 1; + end + endfunction // clogb2 + + localparam MAINT_PRESCALER_DIV = MAINT_PRESCALER_PERIOD/(tCK * 4); // softmc is clocked 4 times slower than the memory interface + localparam MAINT_PRESCALER_WIDTH = clogb2(MAINT_PRESCALER_DIV + 1); + localparam ONE = 1; + + reg maint_prescaler_tick_r_lcl; + reg [MAINT_PRESCALER_WIDTH-1:0] maint_prescaler_r; + reg [MAINT_PRESCALER_WIDTH-1:0] maint_prescaler_ns; + + wire maint_prescaler_tick_ns = (maint_prescaler_r == ONE[MAINT_PRESCALER_WIDTH-1:0]); + + always @(/*AS*/init_calib_complete or maint_prescaler_r + or maint_prescaler_tick_ns) begin + maint_prescaler_ns = maint_prescaler_r; + if (~init_calib_complete || maint_prescaler_tick_ns) + maint_prescaler_ns = MAINT_PRESCALER_DIV[MAINT_PRESCALER_WIDTH-1:0]; + else if (|maint_prescaler_r) + maint_prescaler_ns = maint_prescaler_r - ONE[MAINT_PRESCALER_WIDTH-1:0]; + end + + always @(posedge clk) maint_prescaler_r <= maint_prescaler_ns; + + always @(posedge clk) maint_prescaler_tick_r_lcl <= maint_prescaler_tick_ns; + + localparam tPRDI = 1_000_000; + localparam PERIODIC_RD_TIMER_DIV = tPRDI/MAINT_PRESCALER_PERIOD; + localparam PERIODIC_RD_TIMER_WIDTH = clogb2(PERIODIC_RD_TIMER_DIV + /*idle state*/ 1); + + reg [PERIODIC_RD_TIMER_WIDTH-1:0] periodic_rd_timer_r, periodic_rd_timer; + + always @* begin + periodic_rd_timer = periodic_rd_timer_r; + + if(~init_calib_complete) begin + periodic_rd_timer = {PERIODIC_RD_TIMER_WIDTH{1'b0}}; + end + else if (per_rd_ack || program_process) begin + periodic_rd_timer = PERIODIC_RD_TIMER_DIV[0+:PERIODIC_RD_TIMER_WIDTH]; + end + else if (|periodic_rd_timer_r && maint_prescaler_tick_r_lcl) begin + periodic_rd_timer = periodic_rd_timer_r - ONE[0+:PERIODIC_RD_TIMER_WIDTH]; + end + end //always + + wire periodic_rd_timer_one = maint_prescaler_tick_r_lcl && (periodic_rd_timer_r == ONE[0+:PERIODIC_RD_TIMER_WIDTH]); + + wire periodic_rd_request = ~rst && (/*((PERIODIC_RD_TIMER_DIV != 0) && ~dfi_init_complete) ||*/ + (~per_rd_ack && (per_rd_request || periodic_rd_timer_one))); + + always @(posedge clk) begin + if(~init_calib_complete) + periodic_rd_timer_r <= PERIODIC_RD_TIMER_DIV[0+:PERIODIC_RD_TIMER_WIDTH]; + else + periodic_rd_timer_r <= periodic_rd_timer; + per_rd_request <= periodic_rd_request; + end //always + + // ZQ timebase. Nominally 128 mS + localparam MAINT_PRESCALER_PERIOD_NS = MAINT_PRESCALER_PERIOD / 1000; + localparam tZQI = 128_000_000; + localparam ZQ_TIMER_DIV = tZQI/MAINT_PRESCALER_PERIOD_NS; + localparam ZQ_TIMER_WIDTH = clogb2(ZQ_TIMER_DIV + 1); + + generate + begin : zq_cntrl + reg zq_tick = 1'b0; + + if (ZQ_TIMER_DIV !=0) begin : zq_timer + reg [ZQ_TIMER_WIDTH-1:0] zq_timer_r; + reg [ZQ_TIMER_WIDTH-1:0] zq_timer_ns; + + always @(/*AS*/init_calib_complete or maint_prescaler_tick_r_lcl or zq_tick or zq_timer_r or program_process) begin + zq_timer_ns = zq_timer_r; + if (~init_calib_complete || zq_tick || program_process) + zq_timer_ns = ZQ_TIMER_DIV[ZQ_TIMER_WIDTH-1:0]; + else if (|zq_timer_r && maint_prescaler_tick_r_lcl) + zq_timer_ns = zq_timer_r - ONE[ZQ_TIMER_WIDTH-1:0]; + end + + always @(posedge clk) zq_timer_r <= zq_timer_ns; + + always @(/*AS*/maint_prescaler_tick_r_lcl or zq_timer_r) + zq_tick = (zq_timer_r == ONE[ZQ_TIMER_WIDTH-1:0] && maint_prescaler_tick_r_lcl); + end // zq_timer + + // ZQ request. Set request with timer tick, and when exiting PHY init. Never + // request if ZQ_TIMER_DIV == 0. + begin : zq_request_logic + wire zq_clear = zq_ack; + reg zq_request_r; + wire zq_request_ns = ~rst && ((~init_calib_complete && (ZQ_TIMER_DIV != 0)) || + (zq_request_r && ~zq_clear) || zq_tick); + + always @(posedge clk) zq_request_r <= zq_request_ns; + + always @(/*AS*/init_calib_complete or zq_request_r) + zq_request = init_calib_complete && zq_request_r; + end // zq_request_logic + end + endgenerate + + localparam tAREF = 7_800; + localparam PERIODIC_REF_TIMER_DIV = tAREF/MAINT_PRESCALER_PERIOD_NS; + localparam PERIODIC_REF_TIMER_WIDTH = clogb2(PERIODIC_REF_TIMER_DIV + /*idle state*/ 1); + + reg [PERIODIC_REF_TIMER_WIDTH-1:0] autoref_timer_r, autoref_timer; + reg aref_switch_ns, aref_switch_r; + + always @* begin + aref_switch_ns = aref_en_valid ? aref_en : aref_switch_r; + pr_ref_request = `LOW; + autoref_timer = autoref_timer_r; + if(~aref_switch_r || ~init_calib_complete) + autoref_timer = PERIODIC_REF_TIMER_DIV[PERIODIC_REF_TIMER_WIDTH-1:0]; + else begin + if(autoref_timer_r > 1 && maint_prescaler_tick_r_lcl) // you were here + autoref_timer = autoref_timer_r - ONE[PERIODIC_REF_TIMER_WIDTH-1:0]; + else if(autoref_timer_r == 1) begin + pr_ref_request = `HIGH; + if(ref_ack) + autoref_timer = PERIODIC_REF_TIMER_DIV[PERIODIC_REF_TIMER_WIDTH-1:0]; + end + end + end + + always @(posedge clk) begin + if(rst) begin + aref_switch_r <= `LOW; + autoref_timer_r = PERIODIC_REF_TIMER_DIV[PERIODIC_REF_TIMER_WIDTH-1:0]; + end + else begin + aref_switch_r <= aref_switch_ns; + autoref_timer_r <= autoref_timer; + end + end + +endmodule diff --git a/sources/hdl/verilog/pop_count4.v b/sources/hdl/verilog/pop_count4.v new file mode 100644 index 0000000..80bea46 --- /dev/null +++ b/sources/hdl/verilog/pop_count4.v @@ -0,0 +1,68 @@ +`timescale 1ns / 1ps +////////////////////////////////////////////////////////////////////////////////// +// Company: +// Engineer: +// +// Create Date: 12/19/2018 10:51:52 AM +// Design Name: +// Module Name: pop_count4 +// Project Name: +// Target Devices: +// Tool Versions: +// Description: +// +// Dependencies: +// +// Revision: +// Revision 0.01 - File Created +// Additional Comments: +// +////////////////////////////////////////////////////////////////////////////////// + + +module pop_count4( + input [3:0] in, + output [2:0] out + ); + + reg [2:0] out_r; + + always @* begin + out_r = 3'd0; + case(in) + 4'b0000: + out_r = 3'd0; + 4'b0001: + out_r = 3'd1; + 4'b0010: + out_r = 3'd1; + 4'b0011: + out_r = 3'd2; + 4'b0100: + out_r = 3'd1; + 4'b0101: + out_r = 3'd2; + 4'b0110: + out_r = 3'd2; + 4'b0111: + out_r = 3'd3; + 4'b1000: + out_r = 3'd1; + 4'b1001: + out_r = 3'd2; + 4'b1010: + out_r = 3'd2; + 4'b1011: + out_r = 3'd3; + 4'b1100: + out_r = 3'd2; + 4'b1101: + out_r = 3'd3; + 4'b1111: + out_r = 3'd4; + endcase + end + + assign out = out_r; + +endmodule diff --git a/sources/hdl/verilog/pre_decode.v b/sources/hdl/verilog/pre_decode.v new file mode 100644 index 0000000..7621b24 --- /dev/null +++ b/sources/hdl/verilog/pre_decode.v @@ -0,0 +1,39 @@ +`include "parameters.vh" +`include "encoding.vh" + +/* + * This combinational logic will output + * whether or not an instruction is a branch + * or not, for now. + */ +module pre_decode( + input [`INSTR_WIDTH-1:0] instruction, + // available readback_fifo entries in terms of read count + input [11:0] buffer_space, + output [11:0] read_size, + output is_branch, + output is_end, + output is_ddr_start, + output need_flush, + output is_sleep + ); + + // When an instruction is non_ddr and has + // is_branch flag set + assign is_branch = instruction[`BRANCH_OFFSET] + && ~instruction[`DDR_OFFSET]; + + assign is_end = &(~instruction); + + // encountered a ddr command segments + assign is_ddr_start = instruction[`INFO_OFFSET] + && ~instruction[`DDR_OFFSET]; + + assign read_size = instruction[9:0]; + // if there are more reads than we can buffer + assign need_flush = is_ddr_start && (read_size > buffer_space); + + assign is_sleep = instruction[`BRANCH_OFFSET] && + instruction[`FU_CODE_OFFSET +: 8] == `SLEEP; + +endmodule diff --git a/sources/hdl/verilog/readback_engine.v b/sources/hdl/verilog/readback_engine.v new file mode 100644 index 0000000..496d83e --- /dev/null +++ b/sources/hdl/verilog/readback_engine.v @@ -0,0 +1,244 @@ +`include "parameters.vh" + +// Process data coming from DRAM before sending it to the host. +module readback_engine( + + // common signals + input clk, + input rst, + + // other control signals + input flush, + input read_seq_incoming, // next few instructions will read from DRAM + input [11:0] incoming_reads, // how many reads next few instructions will issue + output[11:0] buffer_space, // remaining buffer size + input switch_mode, + + // DRAM <-> engine if + input [511:0] rd_data, + input rd_valid, + + input per_rd_init, + input per_zq_init, + input per_ref_init, + + // engine <-> regfile if + input [511:0] ddr_wdata, // to compare read data against + + // readback <-> XDMA if + output [`XDMA_AXI_DATA_WIDTH-1:0] c2h_tdata_0, + output c2h_tlast_0, + output c2h_tvalid_0, + input c2h_tready_0, + output [`XDMA_AXI_DATA_WIDTH/8-1:0] c2h_tkeep_0 + + ); + + + localparam READ_MODE = 0; + localparam DIFF_MODE = 1; + reg mode_r, mode_ns; // Switch between diff count and read modes + + reg rd_valid_r; + reg ignore_read_r, ignore_read_ns; + reg ignore_flush_r, ignore_flush_ns; + + // Popcount computation part + reg[511:0] read_diff; + reg diff_valid; + always @(posedge clk) begin + if(rst) begin + read_diff <= 512'bX; + diff_valid <= `LOW; + end + read_diff <= rd_valid ? rd_data ^ ddr_wdata : read_diff; + diff_valid <= rd_valid && ~ignore_read_r && mode_r == DIFF_MODE ? `HIGH : `LOW; + end + + genvar pcs; // popcount modules + + wire[2:0] pc_out [127:0]; + reg[3:0] pc_out_l2 [63:0]; + reg[4:0] pc_out_l3 [31:0]; + reg[5:0] pc_out_l4 [15:0]; + reg[6:0] pc_out_l5 [7:0]; + reg[7:0] pc_out_l6 [3:0]; + reg[8:0] pc_out_l7 [1:0]; + reg[15:0] pop_count_value; + reg pop_count_valid; + + generate + for(pcs = 0 ; pcs < 128 ; pcs = pcs + 1) begin: gen_pcs + pop_count4 pci + ( + .in(read_diff[pcs*4 +: 4]), + .out(pc_out[pcs]) + ); + end + endgenerate + + integer l1, l2, l3, l4, l5, l6; + always @* begin + for(l1 = 0 ; l1 < 64 ; l1 = l1+1) + pc_out_l2[l1] = pc_out[2*l1] + pc_out[2*l1+1]; + for(l2 = 0 ; l2 < 32 ; l2 = l2+1) + pc_out_l3[l2] = pc_out_l2[2*l2] + pc_out_l2[2*l2+1]; + for(l3 = 0 ; l3 < 16 ; l3 = l3+1) + pc_out_l4[l3] = pc_out_l3[2*l3] + pc_out_l3[2*l3+1]; + for(l4 = 0 ; l4 < 8 ; l4 = l4+1) + pc_out_l5[l4] = pc_out_l4[2*l4] + pc_out_l4[2*l4+1]; + for(l5 = 0 ; l5 < 4 ; l5 = l5+1) + pc_out_l6[l5] = pc_out_l5[2*l5] + pc_out_l5[2*l5+1]; + for(l6 = 0 ; l6 < 2 ; l6 = l6+1) + pc_out_l7[l6] = pc_out_l6[2*l6] + pc_out_l6[2*l6+1]; + end + + always @(posedge clk) begin + if(rst) begin + pop_count_value <= 16'bX; + pop_count_valid <= `LOW; + end + else begin + pop_count_value <= diff_valid ? pc_out_l7[0] + pc_out_l7[1] : pop_count_value; + pop_count_valid <= diff_valid ? `HIGH : `LOW; + end + end + + wire[511:0] dsr_out; + wire dsr_valid; + // We put popcounted data into a shift register + // to fill up 512 bit I/O fifo. + diff_shift_reg dsr( + .clk(clk), + .rst(rst), + + .in(pop_count_value), + .in_valid(pop_count_valid), + + .flush(flush&~ignore_flush_r), + + .out(dsr_out), + .out_valid(dsr_valid) + ); + // End popcount computation part + + // Count up to 1024 32-byte transfers + reg[9:0] xctr_r; + + reg tlast; // indicating c2h's last transfer + + // We read DQ_WIDTH*DQ_BURST (512 as of now) bits + // from DRAM, and have to pipe 256 bit partitions of + // it to the PCI. We may read data each cycle from + // DRAM and have to buffer some of those. + wire rbf_empty, rbf_rd_valid, fifo_almost_full, fifo_valid; + (*KEEP = "TRUE"*) wire rbf_full; + (*KEEP = "TRUE"*) reg [19:0] dbg_rd_ctr; + rdback_fifo rbf( + .full(rbf_full), + .prog_full(fifo_almost_full), + .empty(rbf_empty), + .wr_en(mode_r == READ_MODE ? rd_valid && ~ignore_read_r: dsr_valid), + // shuffle data because fifo outputs them on wrong order + .din(mode_r == READ_MODE ? {rd_data[255:0],rd_data[511:256]} : {dsr_out[255:0],dsr_out[511:256]}), + .rd_en(c2h_tready_0), + .dout(c2h_tdata_0), + .valid(fifo_valid), + .clk(clk), + .srst(rst) + ); + + reg proc_flush_ns, proc_flush_r; + // we count the remaining space in terms of + // AXI transactions + // e.g. 1024 reads will take up 2048 + reg [11:0] buffer_space_ns, buffer_space_r; + + always @* begin + tlast = `LOW; + ignore_read_ns = ignore_read_r; + ignore_flush_ns = ignore_flush_r; + buffer_space_ns = buffer_space_r; + if(per_rd_init || per_zq_init || per_ref_init) begin + ignore_read_ns = per_rd_init; + ignore_flush_ns = `HIGH; + end + if(rd_valid_r) + ignore_read_ns = `LOW; + proc_flush_ns = proc_flush_r; + if(flush) begin + if(ignore_flush_r) + ignore_flush_ns = `LOW; + else + proc_flush_ns = `HIGH; + end + mode_ns = mode_r; + if(switch_mode) + mode_ns = ~mode_r; + if(&xctr_r && (c2h_tready_0 && c2h_tvalid_0)) begin + tlast = `HIGH; + end + // Send what's remaining in the fifo + // to host with a random length transfer + // (tlast is not based on the counter value) + if(proc_flush_r) begin + if(c2h_tready_0 && rbf_empty && ~dsr_valid) begin + tlast = `HIGH; + proc_flush_ns = `LOW; + end + else + proc_flush_ns = `HIGH; + end + if(read_seq_incoming) begin + if(c2h_tvalid_0 && c2h_tready_0) begin + buffer_space_ns = (buffer_space_r - (incoming_reads << 1)) + 1; + end + else begin + buffer_space_ns = (buffer_space_r - (incoming_reads << 1)); + end + end + else begin + if(c2h_tvalid_0 && c2h_tready_0) begin + if(~(proc_flush_r && rbf_empty && ~dsr_valid)) + buffer_space_ns = buffer_space_r + 1; + end + end + end + + always @(posedge clk) begin + if(rst) begin + dbg_rd_ctr <= 20'b0; + xctr_r <= 15'b0; + proc_flush_r <= `LOW; + mode_r <= READ_MODE; + ignore_read_r <= 1'b0; + ignore_flush_r <= 1'b0; + rd_valid_r <= 1'b0; + buffer_space_r <= 12'd2048; + end + else begin + if(rd_valid && ~ignore_read_r && ~rbf_full) + dbg_rd_ctr <= dbg_rd_ctr + 1'b1; + else + dbg_rd_ctr <= dbg_rd_ctr; + buffer_space_r <= buffer_space_ns; + mode_r <= mode_ns; + rd_valid_r <= rd_valid; + ignore_read_r <= ignore_read_ns; + ignore_flush_r <= ignore_flush_ns; + if(proc_flush_r && tlast) + xctr_r <= 15'b0; + else if(c2h_tready_0 && c2h_tvalid_0) begin + xctr_r <= xctr_r + 1; + end + proc_flush_r <= proc_flush_ns; + end + end + + assign c2h_tkeep_0 = {(`XDMA_AXI_DATA_WIDTH/8){1'b1}}; + assign c2h_tlast_0 = tlast; + assign c2h_tvalid_0 = proc_flush_r && rbf_empty && ~dsr_valid ? `HIGH : fifo_valid; + + assign buffer_space = buffer_space_r >> 1; + +endmodule diff --git a/sources/hdl/verilog/reg_mem.v b/sources/hdl/verilog/reg_mem.v new file mode 100644 index 0000000..a8d37cd --- /dev/null +++ b/sources/hdl/verilog/reg_mem.v @@ -0,0 +1,33 @@ +`timescale 1ns / 1ps + +module reg_mem( + input [9:0] addr, + input clk, + input [31:0] din, + output [31:0] dout, + input en, + input we + ); + + reg [31:0] mem [1023:0]; + + integer i; + initial + begin + for(i=0; i<1024; i=i+1) + mem[i]=0; + end + + reg [31:0] out; + + assign dout=out; + + always @(posedge clk) + begin + if(we) + mem[addr]<=din; + else if(en) + out<=mem[addr]; + end + +endmodule diff --git a/sources/hdl/verilog/register_file.v b/sources/hdl/verilog/register_file.v new file mode 100644 index 0000000..2d3b693 --- /dev/null +++ b/sources/hdl/verilog/register_file.v @@ -0,0 +1,92 @@ +`include "parameters.vh" + +/* + * An 8 read 8 write port reg file + * satisfying ddr pipeline's needs + */ +module register_file( + + // common signals + input clk, + input rst, + + // execute stage <-> reg file if + input rf_wide_wen, + input [3:0] rf_wide_offset, + output [511:0] rf_wide_data, + output [32*8-1:0] rf_rdata, + input [32*8-1:0] rf_wdata, + input [7:0] rf_wen, + input [4*8-1:0] rf_raddr, + input [4*8-1:0] rf_waddr, + output [`COL_WIDTH-1:0] casr, + output [`BANK_WIDTH+`BG_WIDTH-1:0] basr, + output [`ROW_WIDTH-1:0] rasr, + // update stride registers, wen is OH (3'b001 = update rasr) + input [31:0] srf_value, + input [2:0] srf_wen + ); + + // Stride registers + reg [31:0] casr_r, basr_r, rasr_r; + + assign casr = casr_r; + assign basr = basr_r; + assign rasr = rasr_r; + + // Update stride registers + always @(posedge clk) begin + if(srf_wen[0]) begin + casr_r <= srf_value; + end + if(srf_wen[1]) begin + basr_r <= srf_value; + end + if(srf_wen[2]) begin + rasr_r <= srf_value; + end + end + + // General purpose registers + reg [31:0] reg_file [15:0]; + + // split r&w data signals into human readable form + // also drive rdata with appropriate register values + wire [31:0] wdata [7:0]; + wire wen [7:0]; + wire [3:0] waddr [7:0]; + wire [3:0] raddr [7:0]; + genvar i; + generate + for(i = 0 ; i < 8 ; i = i + 1) begin: gen_ports + assign wdata[i] = rf_wdata[32*i +: 32]; + assign wen[i] = rf_wen[i]; + assign waddr[i] = rf_waddr[4*i +: 4]; + assign raddr[i] = rf_raddr[4*i +: 4]; + // TODO will this compile? + assign rf_rdata[32*i +: 32] = raddr[i] == 0 ? casr_r : + raddr[i] == 1 ? basr_r : + raddr[i] == 2 ? rasr_r : reg_file[raddr[i]]; + end + endgenerate + + integer j; + // Write to the register file + always @(posedge clk) begin + for(j = 0 ; j < 8 ; j = j+1) begin: regfile_write + if(wen[j]) + reg_file[waddr[j]] <= wdata[j]; + end + end + + // DDR4 8-burst write data register + reg [511:0] wide_reg; + always @(posedge clk) begin + // TODO this probably won't compile + if(rf_wide_wen) + wide_reg[rf_wide_offset*32 +: 32] <= wdata[0]; + end + + assign rf_wide_data = wide_reg; + +endmodule diff --git a/sources/hdl/verilog/softmc_pipeline.v b/sources/hdl/verilog/softmc_pipeline.v new file mode 100644 index 0000000..07c106e --- /dev/null +++ b/sources/hdl/verilog/softmc_pipeline.v @@ -0,0 +1,178 @@ +`include "parameters.vh" + +module softmc_pipeline( + + // common signals + input clk, + input rst, + + // readback <-> fetch_stage backpressure + + output softmc_end, + output [11:0] read_size, + output read_seq_incoming, + input [11:0] buffer_space, + + // frontend <-> fetch stage interface + output [`IMEM_ADDR_WIDTH-1:0] addr_out, + output valid_out, + input [`INSTR_WIDTH-1:0] data_in, + input valid_in, + input [`IMEM_ADDR_WIDTH-1:0] addr_in, + input ready_out, + + // ddr_pipeline <-> outer DDR module + output [3:0] ddr_write, + output [3:0] ddr_read, + output [3:0] ddr_pre, + output [3:0] ddr_act, + output [3:0] ddr_ref, + output [3:0] ddr_sre, + output [3:0] ddr_srx, + output [3:0] ddr_zq, + output [3:0] ddr_nop, + output [3:0] ddr_ap, + output [3:0] ddr_pall, + output [3:0] ddr_half_bl, + output [4*`BG_WIDTH-1:0] ddr_bg, + output [4*`BANK_WIDTH-1:0] ddr_bank, + output [4*`COL_WIDTH-1:0] ddr_col, + output [4*`ROW_WIDTH-1:0] ddr_row, + output [511:0] ddr_wdata + ); + + wire br_resolve; + wire [`IMEM_ADDR_WIDTH-1:0] br_target; + + wire [`INSTR_WIDTH-1:0] dec_instr; + wire dec_instr_valid; + wire [`IMEM_ADDR_WIDTH-1:0] dec_instr_pc; + + fetch_stage fs( + .clk(clk), + .rst(rst), + + .softmc_end(softmc_end), + .read_size(read_size), + .read_seq_incoming(read_seq_incoming), + .buffer_space(buffer_space), + + + .addr_out(addr_out), + .valid_out(valid_out), + .data_in(data_in), + .valid_in(valid_in), + .addr_in(addr_in), + .ready_out(ready_out), + + .br_resolve(br_resolve), + .br_target(br_target), + + .instr(dec_instr), + .instr_pc(dec_instr_pc), + .instr_valid(dec_instr_valid) + ); + + wire ddr_valid, exe_valid; + wire [`DDR_UOP_WIDTH*4-1:0] ddr_uop; + wire [`EXE_UOP_WIDTH-1:0] exe_uop; + wire [`IMEM_ADDR_WIDTH-1:0] exe_pc; + wire [32*7-1:0] ddr_stat; + + decode_stage ds( + .clk(clk), + .rst(rst), + + .instr(dec_instr), + .instr_pc(dec_instr_pc), + .instr_valid(dec_instr_valid), + + .ddr_valid(ddr_valid), + .exe_valid(exe_valid), + .ddr_uop(ddr_uop), + .exe_uop(exe_uop), + .exe_pc(exe_pc), + .ddr_stat(ddr_stat) + ); + + wire rf_wide_wen; + wire [3:0] rf_wide_offset; + wire [511:0] rf_wide_data; + wire [32*8-1:0] rf_rdata; + wire [32*8-1:0] rf_wdata; + wire [7:0] rf_wen; + wire [4*8-1:0] rf_raddr; + wire [4*8-1:0] rf_waddr; + wire [`COL_WIDTH-1:0] casr; + wire [`BANK_WIDTH+`BG_WIDTH-1:0] basr; + wire [`ROW_WIDTH-1:0] rasr; + wire [31:0] srf_value; + wire [2:0] srf_wen; + + execute_stage es( + .clk(clk), + .rst(rst), + + .br_resolve(br_resolve), + .br_target(br_target), + + .ddr_valid(ddr_valid), + .exe_valid(exe_valid), + .ddr_uop(ddr_uop), + .exe_uop(exe_uop), + .exe_pc(exe_pc), + + .rf_wide_wen(rf_wide_wen), + .rf_wide_offset(rf_wide_offset), + .wide_reg(rf_wide_data), + .rf_rdata(rf_rdata), + .rf_wdata(rf_wdata), + .rf_wen(rf_wen), + .rf_raddr(rf_raddr), + .rf_waddr(rf_waddr), + .casr(casr), + .basr(basr), + .rasr(rasr), + .srf_value(srf_value), + .srf_wen(srf_wen), + .ddr_stat(ddr_stat), + + .ddr_write(ddr_write), + .ddr_read(ddr_read), + .ddr_pre(ddr_pre), + .ddr_act(ddr_act), + .ddr_ref(ddr_ref), + .ddr_sre(ddr_sre), + .ddr_srx(ddr_srx), + .ddr_zq(ddr_zq), + .ddr_nop(ddr_nop), + .ddr_ap(ddr_ap), + .ddr_pall(ddr_pall), + .ddr_half_bl(ddr_half_bl), + .ddr_bg(ddr_bg), + .ddr_bank(ddr_bank), + .ddr_col(ddr_col), + .ddr_row(ddr_row), + .ddr_wdata(ddr_wdata) + ); + + register_file rf( + .clk(clk), + .rst(rst), + + .rf_wide_wen(rf_wide_wen), + .rf_wide_offset(rf_wide_offset), + .rf_wide_data(rf_wide_data), + .rf_rdata(rf_rdata), + .rf_wdata(rf_wdata), + .rf_wen(rf_wen), + .rf_raddr(rf_raddr), + .rf_waddr(rf_waddr), + .casr(casr), + .basr(basr), + .rasr(rasr), + .srf_value(srf_value), + .srf_wen(srf_wen) + ); + +endmodule diff --git a/sources/hdl/verilog/softmc_top.v b/sources/hdl/verilog/softmc_top.v new file mode 100644 index 0000000..c5b1c76 --- /dev/null +++ b/sources/hdl/verilog/softmc_top.v @@ -0,0 +1,838 @@ +`include "parameters.vh" +`include "project.vh" + +`ifdef XUPP3R_x4 + `define XUPP3R +`elsif XUPP3R_x8 + `define XUPP3R +`elsif XUPP3R_x8_1R_UDIMM + `define XUPP3R +`endif + + +module softmc_top #(parameter tCK = 1500, SIM = "false") + ( + // common signals + input c0_sys_clk_p, + input c0_sys_clk_n, + input sys_rst_l, + + // iob <> ddr4 sdram ip signals + output c0_ddr4_act_n, + output [16:0] c0_ddr4_adr, + output [1:0] c0_ddr4_ba, + output [1:0] c0_ddr4_bg, + output [`CKE_WIDTH-1:0] c0_ddr4_cke, + output [`ODT_WIDTH-1:0] c0_ddr4_odt, + output [`CS_WIDTH-1:0] c0_ddr4_cs_n, + output [`CK_WIDTH-1:0] c0_ddr4_ck_t, + output [`CK_WIDTH-1:0] c0_ddr4_ck_c, + output c0_ddr4_reset_n, + `ifdef XUPP3R_x4 + inout [17:0] c0_ddr4_dqs_c, + inout [17:0] c0_ddr4_dqs_t, + inout [71:0] c0_ddr4_dq, + output c0_ddr4_parity, + `elsif XUPP3R_x8 + inout [8:0] c0_ddr4_dm_dbi_n, + inout [71:0] c0_ddr4_dq, + inout [8:0] c0_ddr4_dqs_c, + inout [8:0] c0_ddr4_dqs_t, + output c0_ddr4_parity, + `else + inout [7:0] c0_ddr4_dm_dbi_n, + inout [63:0] c0_ddr4_dq, + inout [7:0] c0_ddr4_dqs_c, + inout [7:0] c0_ddr4_dqs_t, + `endif + // xdma signals + input clk_ref_p, + input clk_ref_n, + input pcie_rst, + output [7:0] pci_exp_txp, + output [7:0] pci_exp_txn, + input [7:0] pci_exp_rxp, + input [7:0] pci_exp_rxn + + ); + + // Frontend control signals + wire softmc_fin; + wire user_rst; + + // Frontend <-> Fetch signals + wire [`IMEM_ADDR_WIDTH-1:0] fr_addr_in; + wire fr_valid_in; + wire [`INSTR_WIDTH-1:0] fr_data_out; + wire fr_valid_out; + wire [`IMEM_ADDR_WIDTH-1:0] fr_addr_out; + wire fr_ready_out; + + // Frontend <-> misc. control signals + wire per_rd_init; + wire per_zq_init; + wire per_ref_init; + wire rbe_switch_mode; + wire toggle_dll; + + // AXI streaming ports + wire [`XDMA_AXI_DATA_WIDTH-1:0] m_axis_h2c_tdata_0,xdma_h2c_tdata_0; + wire m_axis_h2c_tlast_0, xdma_h2c_tlast_0; + wire m_axis_h2c_tvalid_0, xdma_h2c_tvalid_0; + wire m_axis_h2c_tready_0, xdma_h2c_tready_0; + wire [`XDMA_AXI_DATA_WIDTH/8-1:0] m_axis_h2c_tkeep_0, xdma_h2c_tkeep_0; + wire [`XDMA_AXI_DATA_WIDTH-1:0] s_axis_c2h_tdata_0, xdma_c2h_tdata_0; + wire s_axis_c2h_tlast_0, xdma_c2h_tlast_0; + wire s_axis_c2h_tvalid_0, xdma_c2h_tvalid_0; + wire s_axis_c2h_tready_0, xdma_c2h_tready_0; + wire [`XDMA_AXI_DATA_WIDTH/8-1:0] s_axis_c2h_tkeep_0, xdma_c2h_tkeep_0; + + // ddr_pipeline <-> outer module if + wire [3:0] ddr_write; + wire [3:0] ddr_read; + wire [3:0] ddr_pre; + wire [3:0] ddr_act; + wire [3:0] ddr_ref; + wire [3:0] ddr_sre; + wire [3:0] ddr_srx; + wire [3:0] ddr_zq; + wire [3:0] ddr_nop; + wire [3:0] ddr_ap; + wire [3:0] ddr_pall; + wire [3:0] ddr_half_bl; + wire [4*`BG_WIDTH-1:0] ddr_bg; + wire [4*`BANK_WIDTH-1:0] ddr_bank; + wire [4*`COL_WIDTH-1:0] ddr_col; + wire [4*`ROW_WIDTH-1:0] ddr_row; + wire [511:0] ddr_wdata; + + // periodic maintenance signals + wire ddr_maint_read; + + // phy <-> ddr adapter and xdma app signals + // dlltoggler + wire clk_sel = 0; + wire [7:0] dllt_mc_ACT_n; + wire [135:0] dllt_mc_ADR; + wire [15:0] dllt_mc_BA; + wire [15:0] dllt_mc_BG; + wire [7:0] dllt_mc_CKE; + wire [7:0] dllt_mc_CS_n; + wire dllt_done; + // ddr adapter + wire [4:0] dBufAdr; + wire [`DQ_WIDTH*8-1:0] wrData; + wire [`DQ_WIDTH-1:0] wrDataMask; + wire [511:0] rdData; + wire [4:0] rdDataAddr; + wire [0:0] rdDataEn; + wire [0:0] rdDataEnd; + wire [0:0] per_rd_done; + wire [0:0] rmw_rd_done; + wire [4:0] wrDataAddr; + wire [0:0] wrDataEn; + wire [7:0] mc_ACT_n; + wire [135:0] mc_ADR; + wire [15:0] mc_BA; + wire [15:0] mc_BG; + wire [`CKE_WIDTH*8-1:0] mc_CKE; + wire [`CS_WIDTH*8-1:0] mc_CS_n; + wire [`ODT_WIDTH*8-1:0] mc_ODT; + wire [0:0] mcRdCAS; + wire [0:0] mcWrCAS; + wire [0:0] winInjTxn; + wire [0:0] winRmw; + wire [4:0] winBuf; + wire [1:0] winRank; + wire [5:0] tCWL; + wire dbg_clk; + wire c0_wr_rd_complete; + wire c0_ddr4_clk; + wire c0_ddr4_dll_off_clk; + wire ddr4_ui_clk; + wire c0_ddr4_rst; + wire [511:0] dbg_bus; + wire [1:0] mcCasSlot; + wire mcCasSlot2; + wire gt_data_ready; + + wire read_seq_incoming; // next few instructions will read from DRAM + wire [11:0] incoming_reads; // how many reads next few instructions will issue + wire [11:0] buffer_space; // remaining buffer size + + wire sys_rst = ~sys_rst_l; // low active signal + wire c0_init_calib_complete; + + // There is a possibility that these signals are on + // the critical path as observed in + // the previous iteration of SoftMC + reg c0_init_calib_complete_r, sys_rst_r; + wire iq_full, processing_iseq, rdback_fifo_empty; + + always @(posedge c0_ddr4_clk) begin + c0_init_calib_complete_r <= c0_init_calib_complete; + sys_rst_r <= sys_rst; + end + + reg dllt_active = 1'b0; + + `ifdef ENABLE_DLL_TOGGLER + always @(posedge c0_ddr4_clk) begin + if(toggle_dll) begin + dllt_active <= ~dllt_active; + end + if(dllt_done) begin + dllt_active <= ~dllt_active; + end + end + `endif + + `ifdef XUPP3R_x8 + phy_ddr4_x8 phy_ddr4_i( + .sys_rst (sys_rst), + .c0_sys_clk_p (c0_sys_clk_p), + .c0_sys_clk_n (c0_sys_clk_n), + `ifdef ENABLE_DLL_TOGGLER + .c0_ddr4_ui_clk (ddr4_ui_clk), + .addn_ui_clkout1 (c0_ddr4_dll_off_clk), + `else + .c0_ddr4_ui_clk (c0_ddr4_clk), + `endif + .c0_ddr4_ui_clk_sync_rst (c0_ddr4_rst), + .c0_init_calib_complete (c0_init_calib_complete), + .dbg_clk (dbg_clk), + .c0_ddr4_act_n (c0_ddr4_act_n), + .c0_ddr4_adr (c0_ddr4_adr), + .c0_ddr4_ba (c0_ddr4_ba), + .c0_ddr4_bg (c0_ddr4_bg), + .c0_ddr4_cke (c0_ddr4_cke), + .c0_ddr4_odt (c0_ddr4_odt), + .c0_ddr4_cs_n (c0_ddr4_cs_n), + .c0_ddr4_ck_t (c0_ddr4_ck_t), + .c0_ddr4_ck_c (c0_ddr4_ck_c), + .c0_ddr4_reset_n (c0_ddr4_reset_n), + .c0_ddr4_parity (c0_ddr4_parity), + .wrDataMask (wrDataMask), + .c0_ddr4_dm_dbi_n (c0_ddr4_dm_dbi_n), + .c0_ddr4_dq (c0_ddr4_dq), + .c0_ddr4_dqs_c (c0_ddr4_dqs_c), + .c0_ddr4_dqs_t (c0_ddr4_dqs_t), + + .dBufAdr (dBufAdr), + .wrData (wrData), + .rdData (rdData), + .rdDataAddr (rdDataAddr), + .rdDataEn (rdDataEn), + .rdDataEnd (rdDataEnd), + .per_rd_done (per_rd_done), + .rmw_rd_done (rmw_rd_done), + .wrDataAddr (wrDataAddr), + .wrDataEn (wrDataEn), + + .mc_ACT_n (dllt_active ? dllt_mc_ACT_n : mc_ACT_n), + .mc_ADR (dllt_active ? dllt_mc_ADR : mc_ADR), + .mc_BA (dllt_active ? dllt_mc_BA : mc_BA), + .mc_BG (dllt_active ? dllt_mc_BG : mc_BG), + .mc_CKE (dllt_active ? dllt_mc_CKE : mc_CKE), + .mc_CS_n (dllt_active ? dllt_mc_CS_n : mc_CS_n), + .mc_ODT (mc_ODT), + // CAS command slot select. Slot0 is enabled for example design. + .mcCasSlot (dllt_active ? 2'b0 : mcCasSlot), + // CAS slot 2 select. mcCasSlot2 serves a similar purpose as the mcCasSlot[1:0] signal, but mcCasSlot2 is used in timing + // critical logic in the Phy. Slot0 is enabled for example design. + .mcCasSlot2 (dllt_active ? 1'b0 : mcCasSlot2), + .mcRdCAS (dllt_active ? 1'b0 : mcRdCAS), + .mcWrCAS (dllt_active ? 1'b0 : mcWrCAS), + // Optional read command type indication. The winInjTxn signal is set to '0' for example design. + .winInjTxn ({1{1'b0}}), + // Optional read command type indication. The winRmw signal is set to '0' for example design. + .winRmw ({1{1'b0}}), + // Update VT Tracking. The gt_data_ready signal is set to '0' in this example design. + // This signal must be asserted periodically to keep the DQS Gate aligned as voltage and temperature drift. + // For more information, Refer to PG150 document. + .gt_data_ready (gt_data_ready), + .winBuf (winBuf), + .winRank (winRank), + .tCWL (tCWL), + // Debug Port + .dbg_bus (dbg_bus) + ); + `elsif XUPP3R_x4 + phy_ddr4 phy_ddr4_i( + .sys_rst (sys_rst), + .c0_sys_clk_p (c0_sys_clk_p), + .c0_sys_clk_n (c0_sys_clk_n), + + `ifdef ENABLE_DLL_TOGGLER + .c0_ddr4_ui_clk (ddr4_ui_clk), + .addn_ui_clkout1 (c0_ddr4_dll_off_clk), + `else + .c0_ddr4_ui_clk (c0_ddr4_clk), + `endif + .c0_ddr4_ui_clk_sync_rst (c0_ddr4_rst), + .c0_init_calib_complete (c0_init_calib_complete), + .dbg_clk (dbg_clk), + .c0_ddr4_act_n (c0_ddr4_act_n), + .c0_ddr4_adr (c0_ddr4_adr), + .c0_ddr4_ba (c0_ddr4_ba), + .c0_ddr4_bg (c0_ddr4_bg), + .c0_ddr4_cke (c0_ddr4_cke), + .c0_ddr4_odt (c0_ddr4_odt), + .c0_ddr4_cs_n (c0_ddr4_cs_n), + .c0_ddr4_ck_t (c0_ddr4_ck_t), + .c0_ddr4_ck_c (c0_ddr4_ck_c), + .c0_ddr4_reset_n (c0_ddr4_reset_n), + .c0_ddr4_parity (c0_ddr4_parity), + .c0_ddr4_dq (c0_ddr4_dq), + .c0_ddr4_dqs_c (c0_ddr4_dqs_c), + .c0_ddr4_dqs_t (c0_ddr4_dqs_t), + + .dBufAdr (dBufAdr), + .wrData (wrData), + .rdData (rdData), + .rdDataAddr (rdDataAddr), + .rdDataEn (rdDataEn), + .rdDataEnd (rdDataEnd), + .per_rd_done (per_rd_done), + .rmw_rd_done (rmw_rd_done), + .wrDataAddr (wrDataAddr), + .wrDataEn (wrDataEn), + + .mc_ACT_n (dllt_active ? dllt_mc_ACT_n : mc_ACT_n), + .mc_ADR (dllt_active ? dllt_mc_ADR : mc_ADR), + .mc_BA (dllt_active ? dllt_mc_BA : mc_BA), + .mc_BG (dllt_active ? dllt_mc_BG : mc_BG), + .mc_CKE (dllt_active ? dllt_mc_CKE : mc_CKE), + .mc_CS_n (dllt_active ? dllt_mc_CS_n : mc_CS_n), + .mc_ODT (mc_ODT), + // CAS command slot select. Slot0 is enabled for example design. + .mcCasSlot (dllt_active ? 0 : mcCasSlot), + // CAS slot 2 select. mcCasSlot2 serves a similar purpose as the mcCasSlot[1:0] signal, but mcCasSlot2 is used in timing + // critical logic in the Phy. Slot0 is enabled for example design. + .mcCasSlot2 (dllt_active ? 0 : mcCasSlot2), + .mcRdCAS (dllt_active ? 0 : mcRdCAS), + .mcWrCAS (dllt_active ? 0 : mcWrCAS), + // Optional read command type indication. The winInjTxn signal is set to '0' for example design. + .winInjTxn ({1{1'b0}}), + // Optional read command type indication. The winRmw signal is set to '0' for example design. + .winRmw ({1{1'b0}}), + // Update VT Tracking. The gt_data_ready signal is set to '0' in this example design. + // This signal must be asserted periodically to keep the DQS Gate aligned as voltage and temperature drift. + // For more information, Refer to PG150 document. + .gt_data_ready (gt_data_ready), + .winBuf (winBuf), + .winRank (winRank), + .tCWL (tCWL), + // Debug Port + .dbg_bus (dbg_bus) + ); + `elsif XUPP3R_x8_1R_UDIMM + phy_ddr4_udimm phy_ddr4_i( + .sys_rst (sys_rst), + .c0_sys_clk_p (c0_sys_clk_p), + .c0_sys_clk_n (c0_sys_clk_n), + `ifdef ENABLE_DLL_TOGGLER + .c0_ddr4_ui_clk (ddr4_ui_clk), + .addn_ui_clkout1 (c0_ddr4_dll_off_clk), + `else + .c0_ddr4_ui_clk (c0_ddr4_clk), + `endif + .c0_ddr4_ui_clk_sync_rst (c0_ddr4_rst), + .c0_init_calib_complete (c0_init_calib_complete), + .dbg_clk (dbg_clk), + .c0_ddr4_act_n (c0_ddr4_act_n), + .c0_ddr4_adr (c0_ddr4_adr), + .c0_ddr4_ba (c0_ddr4_ba), + .c0_ddr4_bg (c0_ddr4_bg), + .c0_ddr4_cke (c0_ddr4_cke), + .c0_ddr4_odt (c0_ddr4_odt), + .c0_ddr4_cs_n (c0_ddr4_cs_n), + .c0_ddr4_ck_t (c0_ddr4_ck_t), + .c0_ddr4_ck_c (c0_ddr4_ck_c), + .c0_ddr4_reset_n (c0_ddr4_reset_n), + .wrDataMask (wrDataMask), + .c0_ddr4_dm_dbi_n (c0_ddr4_dm_dbi_n), + .c0_ddr4_dq (c0_ddr4_dq), + .c0_ddr4_dqs_c (c0_ddr4_dqs_c), + .c0_ddr4_dqs_t (c0_ddr4_dqs_t), + + .dBufAdr (dBufAdr), + .wrData (wrData), + .rdData (rdData), + .rdDataAddr (rdDataAddr), + .rdDataEn (rdDataEn), + .rdDataEnd (rdDataEnd), + .per_rd_done (per_rd_done), + .rmw_rd_done (rmw_rd_done), + .wrDataAddr (wrDataAddr), + .wrDataEn (wrDataEn), + + .mc_ACT_n (dllt_active ? dllt_mc_ACT_n : mc_ACT_n), + .mc_ADR (dllt_active ? dllt_mc_ADR : mc_ADR), + .mc_BA (dllt_active ? dllt_mc_BA : mc_BA), + .mc_BG (dllt_active ? dllt_mc_BG : mc_BG), + .mc_CKE (dllt_active ? dllt_mc_CKE : mc_CKE), + .mc_CS_n (dllt_active ? dllt_mc_CS_n : mc_CS_n), + .mc_ODT (mc_ODT), + // CAS command slot select. Slot0 is enabled for example design. + .mcCasSlot (dllt_active ? 2'b0 : mcCasSlot), + // CAS slot 2 select. mcCasSlot2 serves a similar purpose as the mcCasSlot[1:0] signal, but mcCasSlot2 is used in timing + // critical logic in the Phy. Slot0 is enabled for example design. + .mcCasSlot2 (dllt_active ? 1'b0 : mcCasSlot2), + .mcRdCAS (dllt_active ? 1'b0 : mcRdCAS), + .mcWrCAS (dllt_active ? 1'b0 : mcWrCAS), + // Optional read command type indication. The winInjTxn signal is set to '0' for example design. + .winInjTxn ({1{1'b0}}), + // Optional read command type indication. The winRmw signal is set to '0' for example design. + .winRmw ({1{1'b0}}), + // Update VT Tracking. The gt_data_ready signal is set to '0' in this example design. + // This signal must be asserted periodically to keep the DQS Gate aligned as voltage and temperature drift. + // For more information, Refer to PG150 document. + .gt_data_ready (gt_data_ready), + .winBuf (winBuf), + .winRank (winRank), + .tCWL (tCWL), + // Debug Port + .dbg_bus (dbg_bus) + ); + `else + phy_ddr4 phy_ddr4_i( + .sys_rst (sys_rst), + .c0_sys_clk_p (c0_sys_clk_p), + .c0_sys_clk_n (c0_sys_clk_n), + + `ifdef ENABLE_DLL_TOGGLER + .c0_ddr4_ui_clk (ddr4_ui_clk), + .addn_ui_clkout1 (c0_ddr4_dll_off_clk), + `else + .c0_ddr4_ui_clk (c0_ddr4_clk), + `endif + .c0_ddr4_ui_clk_sync_rst (c0_ddr4_rst), + .c0_init_calib_complete (c0_init_calib_complete), + .dbg_clk (dbg_clk), + .c0_ddr4_act_n (c0_ddr4_act_n), + .c0_ddr4_adr (c0_ddr4_adr), + .c0_ddr4_ba (c0_ddr4_ba), + .c0_ddr4_bg (c0_ddr4_bg), + .c0_ddr4_cke (c0_ddr4_cke), + .c0_ddr4_odt (c0_ddr4_odt), + .c0_ddr4_cs_n (c0_ddr4_cs_n), + .c0_ddr4_ck_t (c0_ddr4_ck_t), + .c0_ddr4_ck_c (c0_ddr4_ck_c), + .c0_ddr4_reset_n (c0_ddr4_reset_n), + .wrDataMask (wrDataMask), + .c0_ddr4_dm_dbi_n (c0_ddr4_dm_dbi_n), + .c0_ddr4_dq (c0_ddr4_dq), + .c0_ddr4_dqs_c (c0_ddr4_dqs_c), + .c0_ddr4_dqs_t (c0_ddr4_dqs_t), + + .dBufAdr (dBufAdr), + .wrData (wrData), + .rdData (rdData), + .rdDataAddr (rdDataAddr), + .rdDataEn (rdDataEn), + .rdDataEnd (rdDataEnd), + .per_rd_done (per_rd_done), + .rmw_rd_done (rmw_rd_done), + .wrDataAddr (wrDataAddr), + .wrDataEn (wrDataEn), + + .mc_ACT_n (dllt_active ? dllt_mc_ACT_n : mc_ACT_n), + .mc_ADR (dllt_active ? dllt_mc_ADR : mc_ADR), + .mc_BA (dllt_active ? dllt_mc_BA : mc_BA), + .mc_BG (dllt_active ? dllt_mc_BG : mc_BG), + .mc_CKE (dllt_active ? dllt_mc_CKE : mc_CKE), + .mc_CS_n (dllt_active ? dllt_mc_CS_n : mc_CS_n), + .mc_ODT (mc_ODT), + // CAS command slot select. Slot0 is enabled for example design. + .mcCasSlot (dllt_active ? 0 : mcCasSlot), + // CAS slot 2 select. mcCasSlot2 serves a similar purpose as the mcCasSlot[1:0] signal, but mcCasSlot2 is used in timing + // critical logic in the Phy. Slot0 is enabled for example design. + .mcCasSlot2 (dllt_active ? 0 : mcCasSlot2), + .mcRdCAS (dllt_active ? 0 : mcRdCAS), + .mcWrCAS (dllt_active ? 0 : mcWrCAS), + // Optional read command type indication. The winInjTxn signal is set to '0' for example design. + .winInjTxn ({1{1'b0}}), + // Optional read command type indication. The winRmw signal is set to '0' for example design. + .winRmw ({1{1'b0}}), + // Update VT Tracking. The gt_data_ready signal is set to '0' in this example design. + // This signal must be asserted periodically to keep the DQS Gate aligned as voltage and temperature drift. + // For more information, Refer to PG150 document. + .gt_data_ready (gt_data_ready), + .winBuf (winBuf), + .winRank (winRank), + .tCWL (tCWL), + // Debug Port + .dbg_bus (dbg_bus) + ); + `endif + + softmc_pipeline pipeline( + .clk(c0_ddr4_clk), + .rst(c0_ddr4_rst || user_rst), + + .softmc_end(softmc_fin), + .read_size(incoming_reads), + .read_seq_incoming(read_seq_incoming), + .buffer_space(buffer_space), + + .addr_out(fr_addr_in), + .valid_out(fr_valid_in), + .data_in(fr_data_out), + .valid_in(fr_valid_out), + .addr_in(fr_addr_out), + .ready_out(fr_ready_out), + + .ddr_write(ddr_write), + .ddr_read(ddr_read), + .ddr_pre(ddr_pre), + .ddr_act(ddr_act), + .ddr_ref(ddr_ref), + .ddr_sre(ddr_sre), + .ddr_srx(ddr_srx), + .ddr_zq(ddr_zq), + .ddr_nop(ddr_nop), + .ddr_ap(ddr_ap), + .ddr_pall(ddr_pall), + .ddr_half_bl(ddr_half_bl), + .ddr_bg(ddr_bg), + .ddr_bank(ddr_bank), + .ddr_col(ddr_col), + .ddr_row(ddr_row), + .ddr_wdata(ddr_wdata) + ); + + `ifdef ENABLE_DLL_TOGGLER + //BUFGMUX:GeneralClockMuxBuffer + //UltraScale + //XilinxHDLLibrariesGuide, version2014.4 + BUFGMUX#(.CLK_SEL_TYPE("SYNC") //ASYNC,SYNC + )BUFGMUX_inst( + .O(c0_ddr4_clk), //1-bitoutput:Clockoutput + .I0(ddr4_ui_clk), //1-bitinput:Clockinput(S=0) + .I1(c0_ddr4_dll_off_clk), //1-bitinput:Clockinput(S=1) + .S(clk_sel) //1-bitinput:Clockselect + ); + //End of BUFGMUX_inst instantiation + `endif + + + wire frontend_ready; + + frontend #(.SIM_MEM(SIM)) frontend( + .clk(c0_ddr4_clk), + .rst(c0_ddr4_rst), + + .init_calib_complete(c0_init_calib_complete_r), + .softmc_fin(softmc_fin), + .user_rst(user_rst), + + .dllt_begin(toggle_dll), + + // indicates read_back unit is ready for the next iseq + .frontend_ready(frontend_ready), + + // frontend <-> fetch stage if + .addr_in(fr_addr_in), + .valid_in(fr_valid_in), + .data_out(fr_data_out), + .valid_out(fr_valid_out), + .addr_out(fr_addr_out), + .ready_in(fr_ready_out), + + // frontend <-> xdma interface + .h2c_tdata_0(m_axis_h2c_tdata_0), + .h2c_tlast_0(m_axis_h2c_tlast_0), + .h2c_tvalid_0(m_axis_h2c_tvalid_0), + .h2c_tready_0(m_axis_h2c_tready_0), + .h2c_tkeep_0(m_axis_h2c_tkeep_0), + + .per_rd_init(per_rd_init), + .per_zq_init(per_zq_init), + .per_ref_init(per_ref_init), + .rbe_switch_mode(rbe_switch_mode) + ); + + ddr4_adapter#( + `ifdef XUPP3R + `ifdef XUPP3R_x8_1R_UDIMM + .DQ_WIDTH(64) + `else + .DQ_WIDTH(72) + `endif + `endif + ) ddr4_adapter + ( + .clk(c0_ddr4_clk), + .rst(c0_ddr4_rst || user_rst), + .init_calib_complete(c0_init_calib_complete_r), + //.io_config_strobe, + //.io_config, + .dBufAdr(dBufAdr), // Reserved. Should be tied low. + .wrData(wrData), // DRAM write data. There are 8 bits for each DQ lane on the DRAM bus. + .wrDataMask(wrDataMask),// DRAM write DM/DBI port.There is one bit for each byte of the wrData port. + .wrDataEn(wrDataEn), // Write data Enable. The Phy will assert this port for one cycle for each write CAS command. + .mc_ACT_n(mc_ACT_n), // DRAM ACT_n command signal for four DRAM clock cycles. + .mc_ADR(mc_ADR), // DRAM address. There are 8 bits in the fabric interface for each address bit on the DRAM bus. + .mc_BA(mc_BA), // DRAM bank address. 8 bits for each DRAM bank address. + .mc_BG(mc_BG), // DRAM bank group address. + .mc_CS_n(mc_CS_n), // DRAM CS_n + .mc_CKE(mc_CKE), // DRAM CKE + //.mc_ODT(mc_ODT), // DRAM ODT + .mcRdCAS(mcRdCAS), // Read CAS command issued. + .mcWrCAS(mcWrCAS), // Write CAS command issued. + .winRank(winRank), // Target rank for CAS commands. This value indicates which rank a CAS command is issued to. + .winBuf(winBuf), // Optional control signal. When either mcRdCAS or mcWrCAS is asserted, the Phy will store the value on the winBuf signal. + //.rdData(rdData), // DRAM read data. + .rdDataEn(rdDataEn), // Read data valid. This signal asserts for one fabric cycle for each completed read operation. + .rdDataEnd(rdDataEnd), // Unused. Tied high. + .mcCasSlot(mcCasSlot), + .mcCasSlot2(mcCasSlot2), + .gt_data_ready(gt_data_ready), + .ddr_write(ddr_write), + .ddr_read(ddr_read), + .ddr_pre(ddr_pre), + .ddr_act(ddr_act), + .ddr_ref(ddr_ref), + .ddr_sre(ddr_sre), + .ddr_srx(ddr_srx), + .ddr_zq(ddr_zq), + .ddr_nop(ddr_nop), + .ddr_ap(ddr_ap), + .ddr_pall(ddr_pall), + .ddr_half_bl(ddr_half_bl), + .ddr_bg(ddr_bg), + .ddr_bank(ddr_bank), + .ddr_col(ddr_col), + .ddr_row(ddr_row), + .ddr_wdata(ddr_wdata), + + .ddr_maint_read(per_rd_init) + ); + `ifdef XUPP3R_x8 + localparam ODTWRDEL = 5'd9; + localparam ODTWRDUR = 4'd6; + localparam ODTWRODEL = 5'd9; + localparam ODTWRODUR = 4'd6; + localparam ODTRDDEL = 5'd10; + localparam ODTRDDUR = 4'd6; + localparam ODTRDODEL = 5'd9; + localparam ODTRDODUR = 4'd6; + localparam ODTNOP = 16'h0000; + localparam ODTWR = 16'h0021; + localparam ODTRD = 16'h0012; + `elsif XUPP3R_x8_1R_UDIMM + localparam ODTWRDEL = 5'd9; + localparam ODTWRDUR = 4'd6; + localparam ODTWRODEL = 5'd9; + localparam ODTWRODUR = 4'd6; + localparam ODTRDDEL = 5'd9; + localparam ODTRDDUR = 4'd6; + localparam ODTRDODEL = 5'd9; + localparam ODTRDODUR = 4'd6; + localparam ODTNOP = 16'h0000; + localparam ODTWR = 16'h0001; + localparam ODTRD = 16'h0000; + `else + localparam ODTWRDEL = 5'd11; + localparam ODTWRDUR = 4'd6; + localparam ODTWRODEL = 5'd9; + localparam ODTWRODUR = 4'd6; + localparam ODTRDDEL = 5'd11; + localparam ODTRDDUR = 4'd6; + localparam ODTRDODEL = 5'd9; + localparam ODTRDODUR = 4'd6; + localparam ODTNOP = 16'h0000; + localparam ODTWR = 16'h0001; + localparam ODTRD = 16'h0000; + `endif + + wire tranSentC; + assign tranSentC = mcRdCAS | mcWrCAS; + + //synthesis translate_on + //******************************************************************************* + ddr4_mc_odt # ( + .ODTWR (ODTWR) + ,.ODTWRDEL (ODTWRDEL) + ,.ODTWRDUR (ODTWRDUR) + ,.ODTWRODEL (ODTWRODEL) + ,.ODTWRODUR (ODTWRODUR) + + ,.ODTRD (ODTRD) + ,.ODTRDDEL (ODTRDDEL) + ,.ODTRDDUR (ODTRDDUR) + ,.ODTRDODEL (ODTRDODEL) + ,.ODTRDODUR (ODTRDODUR) + + ,.ODTNOP (ODTNOP) + ,.ODTBITS (`ODT_WIDTH) + ,.TCQ (0.1) + )u_ddr_tb_odt( + .clk (c0_ddr4_clk) + ,.rst (c0_ddr4_rst) + ,.mc_ODT (mc_ODT) + ,.casSlot (mcCasSlot) + ,.casSlot2 (mcCasSlot2) + ,.rank (winRank) + ,.winRead (mcRdCAS) + ,.winWrite (mcWrCAS) + ,.tranSentC (tranSentC) + ); + + wire sys_clk, sys_clk_gt; + wire [2:0] msi_vector_width; + wire msi_enable; + wire user_lnk_up, usr_irq_req, usr_irq_ack; + `ifdef XUPP3R + IBUFDS_GTE4 refclk_ibuf (.O(sys_clk_gt), .ODIV2(sys_clk), .I(clk_ref_p), .CEB(1'b0), .IB(clk_ref_n)); + `else + IBUFDS_GTE3 # (.REFCLK_HROW_CK_SEL(2'b01)) refclk_ibuf (.O(sys_clk_gt), .ODIV2(sys_clk), .I(clk_ref_p), .CEB(1'b0), .IB(clk_ref_n)); + `endif + wire axi_clk, axi_rst; + + xdma xdma_i + ( + //---------------------------------------------------------------------------------------// + // PCI Express (pci_exp) Interface // + //---------------------------------------------------------------------------------------// + .sys_rst_n ( pcie_rst ), + .sys_clk ( sys_clk ), + .sys_clk_gt ( sys_clk_gt), + + // Tx + .pci_exp_txn ( pci_exp_txn ), + .pci_exp_txp ( pci_exp_txp ), + + // Rx + .pci_exp_rxn ( pci_exp_rxn ), + .pci_exp_rxp ( pci_exp_rxp ), + + // AXI streaming ports + .s_axis_c2h_tdata_0(xdma_c2h_tdata_0), + .s_axis_c2h_tlast_0(xdma_c2h_tlast_0), + .s_axis_c2h_tvalid_0(xdma_c2h_tvalid_0), + .s_axis_c2h_tready_0(xdma_c2h_tready_0), + .s_axis_c2h_tkeep_0(xdma_c2h_tkeep_0), + .m_axis_h2c_tdata_0(xdma_h2c_tdata_0), + .m_axis_h2c_tlast_0(xdma_h2c_tlast_0), + .m_axis_h2c_tvalid_0(xdma_h2c_tvalid_0), + .m_axis_h2c_tready_0(xdma_h2c_tready_0), + .m_axis_h2c_tkeep_0(xdma_h2c_tkeep_0), + + .usr_irq_req (1'b0), + .usr_irq_ack (usr_irq_ack), + .msi_enable (msi_enable), + .msi_vector_width (msi_vector_width), + + + // Config managemnet interface + .cfg_mgmt_addr ( 19'b0 ), + .cfg_mgmt_write ( 1'b0 ), + .cfg_mgmt_write_data ( 32'b0 ), + .cfg_mgmt_byte_enable ( 4'b0 ), + .cfg_mgmt_read ( 1'b0 ), + .cfg_mgmt_read_data (), + .cfg_mgmt_read_write_done (), + `ifndef XUPP3R + .cfg_mgmt_type1_cfg_reg_access ( 1'b0 ), + //---------- Shared Logic Internal ------------------------- + .int_qpll1lock_out ( ), + .int_qpll1outrefclk_out ( ), + .int_qpll1outclk_out ( ), + `endif + + //-- AXI Global + .axi_aclk (axi_clk), // AXI i-face clock driven from pcie clk + .axi_aresetn (axi_rst), // reset synchronous to axi_clk + + .user_lnk_up ( user_lnk_up ) + ); + + // Clock converter for the c2h interface + axis_clock_converter axis_clk_conv_i0 + ( + .s_axis_tvalid(s_axis_c2h_tvalid_0), + .s_axis_tlast(s_axis_c2h_tlast_0), + .s_axis_tdata(s_axis_c2h_tdata_0), + .s_axis_tkeep(s_axis_c2h_tkeep_0), + .s_axis_tready(s_axis_c2h_tready_0), + .m_axis_tvalid(xdma_c2h_tvalid_0), + .m_axis_tlast(xdma_c2h_tlast_0), + .m_axis_tdata(xdma_c2h_tdata_0), + .m_axis_tkeep(xdma_c2h_tkeep_0), + .m_axis_tready(xdma_c2h_tready_0), + .s_axis_aresetn(~c0_ddr4_rst), + .s_axis_aclk(c0_ddr4_clk), + .m_axis_aresetn(axi_rst), + .m_axis_aclk(axi_clk) + ); + + // Clock converter for the h2c interface + axis_clock_converter axis_clk_conv_i1 + ( + .m_axis_tvalid(m_axis_h2c_tvalid_0), + .m_axis_tlast(m_axis_h2c_tlast_0), + .m_axis_tdata(m_axis_h2c_tdata_0), + .m_axis_tkeep(m_axis_h2c_tkeep_0), + .m_axis_tready(m_axis_h2c_tready_0), + .s_axis_tvalid(xdma_h2c_tvalid_0), + .s_axis_tlast(xdma_h2c_tlast_0), + .s_axis_tdata(xdma_h2c_tdata_0), + .s_axis_tkeep(xdma_h2c_tkeep_0), + .s_axis_tready(xdma_h2c_tready_0), + .m_axis_aresetn(~c0_ddr4_rst), + .m_axis_aclk(c0_ddr4_clk), + .s_axis_aresetn(axi_rst), + .s_axis_aclk(axi_clk) + ); + + readback_engine rbe( + + // common signals + .clk(c0_ddr4_clk), + .rst(c0_ddr4_rst || user_rst), + + // other ctrl signals + .flush(frontend_ready), + .switch_mode(rbe_switch_mode), + .read_seq_incoming(read_seq_incoming), // next few instructions will read from DRAM + .incoming_reads(incoming_reads), // how many reads next few instructions will issue + .buffer_space(buffer_space), // remaining buffer size + // DRAM <-> engine if + .rd_data(rdData), + .rd_valid(rdDataEn), + + // rbe <-> rf interface + .ddr_wdata(ddr_wdata), + + .per_rd_init(per_rd_init), + .per_zq_init(per_zq_init), + .per_ref_init(per_ref_init), + + // rbe <-> xdma if + .c2h_tdata_0(s_axis_c2h_tdata_0), + .c2h_tlast_0(s_axis_c2h_tlast_0), + .c2h_tvalid_0(s_axis_c2h_tvalid_0), + .c2h_tready_0(s_axis_c2h_tready_0), + .c2h_tkeep_0(s_axis_c2h_tkeep_0) + + ); + + `ifdef ENABLE_DLL_TOGGLER + dll_toggler dllt + ( + .clk(c0_ddr4_clk), + .rst(c0_ddr4_rst || user_rst || ~c0_init_calib_complete_r), + .toggle_valid(toggle_dll), + .mc_ACT_n(dllt_mc_ACT_n), // DRAM ACT_n command signal for four DRAM clock cycles. + .mc_ADR(dllt_mc_ADR), // DRAM address. There are 8 bits in the fabric interface for each address bit on the DRAM bus. + .mc_BA(dllt_mc_BA), // DRAM bank address. 8 bits for each DRAM bank address. + .mc_BG(dllt_mc_BG), // DRAM bank group address. + .mc_CS_n(dllt_mc_CS_n), // DRAM CS_n + .mc_CKE(dllt_mc_CKE), + .clk_sel(clk_sel), + .dllt_done(dllt_done) + ); + `endif +endmodule diff --git a/sources/scripts/generate.tcl b/sources/scripts/generate.tcl new file mode 100644 index 0000000..375890b --- /dev/null +++ b/sources/scripts/generate.tcl @@ -0,0 +1,111 @@ +# Returns all directory names in a given directory (non-recursive) +proc get_dir_names {path} { + set dirs [glob -directory "$path" -type d *] + set dir_names {} + foreach project $dirs { + lappend dir_names [string map "$path/ \"\"" $project] + } + return $dir_names +} + +# Version safe wait procedure for a single run. wait_on_run changed to wait_on_runs after Vivado 2021.2 +proc safe_wait { run } { + if { [catch wait_on_runs $run] } { + wait_on_run $run + } +} + +# Version safe wait procedure for multiple runs. wait_on_run changed to wait_on_runs after Vivado 2021.2 +proc safe_wait_multiple { run_list } { + if { [catch wait_on_runs $run_list] } { + foreach run $run_list { wait_on_run $run } + } +} + +# Check required args +if { $argc != 1 } { + puts "Please provide <number_of_threads> to script." + puts "For example, vivado -mode batch generate.tcl -tclargs 12" + exit 0 +} + +# Set path variables +set PROJECT_PATH [pwd] +set PROJECT_NAME [string map ".xpr \"\" $PROJECT_PATH/ \"\"" [glob -directory $PROJECT_PATH *.xpr]] +set PROJECT_IP_PATH "$PROJECT_PATH/$PROJECT_NAME.srcs/sources_1/ip" +set NUM_JOBS [lindex $argv 0] + +# Check if project exists +if { ![file exist $PROJECT_PATH] } { + puts "Project not found on $PROJECT_PATH." + exit 0 +} + +# Open the project +open_project "$PROJECT_PATH/$PROJECT_NAME.xpr" + +update_compile_order -fileset sources_1 + +# Reset previous runs +reset_project + +# IP run variables +set ip_list [get_ips] +set ip_runs {} +set phy_list {} +foreach ip $ip_list { + if {[string first "phy_" $ip] != -1} { + lappend phy_list "$ip" + } + lappend ip_runs "${ip}_synth_1" +} + + +# Reset ourput products of all IPs +reset_target all $ip_list + +# Set fileset to main sources +update_compile_order -fileset sources_1 + +# Re-generate all output products +generate_target all $ip_list + +foreach ip $ip_list { + create_ip_run [get_ips $ip] +} + +launch_runs $ip_runs -verbose -jobs $NUM_JOBS +safe_wait_multiple $ip_runs + +# Lock PHY and Apply patches +foreach phy_ip $phy_list { + set_property IS_LOCKED true [get_files "${phy_ip}.xci"] +} + +if { [catch {exec "${PROJECT_PATH}/apply_patches.sh"} result] != 0 } { + puts "Script 'apply_patches.sh' has no return value, assumed exit value 0 (TCL_OK). Check first incase any errors." +} + +# Reset and relaunch PHY runs +foreach phy_ip $phy_list { + reset_run [get_runs "${phy_ip}_synth_1"] +} + +foreach phy_ip $phy_list { + launch_run [get_runs "${phy_ip}_synth_1"] +} + +foreach phy_ip $phy_list { + safe_wait "${phy_ip}_synth_1" +} + +# Synthesis +launch_runs synth_1 -verbose -jobs $NUM_JOBS +safe_wait "synth_1" + +# Implementation and Bitstream generation +launch_runs impl_1 -to_step write_bitstream -verbose -jobs $NUM_JOBS +safe_wait "impl_1" + +# Done +puts "Bitstream generated successfully. Exitting." diff --git a/sources/xdma_driver/.gitignore b/sources/xdma_driver/.gitignore new file mode 100644 index 0000000..951f319 --- /dev/null +++ b/sources/xdma_driver/.gitignore @@ -0,0 +1,9 @@ +xdma/*.cmd +xdma/*.d +xdma/*.o +xdma/*.ko +xdma/*.o.cmd +xdma/*.order +xdma/*.mod +xdma/*.mod.c +xdma/*.symvers
\ No newline at end of file diff --git a/sources/xdma_driver/COPYING b/sources/xdma_driver/COPYING new file mode 100644 index 0000000..3912109 --- /dev/null +++ b/sources/xdma_driver/COPYING @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/sources/xdma_driver/LICENSE b/sources/xdma_driver/LICENSE new file mode 100644 index 0000000..703e647 --- /dev/null +++ b/sources/xdma_driver/LICENSE @@ -0,0 +1,30 @@ +BSD License + +For Xilinx DMA IP software + +Copyright (c) 2016-present, Xilinx, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Xilinx nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/sources/xdma_driver/include/libxdma_api.h b/sources/xdma_driver/include/libxdma_api.h new file mode 100644 index 0000000..00d4355 --- /dev/null +++ b/sources/xdma_driver/include/libxdma_api.h @@ -0,0 +1,130 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef __XDMA_BASE_API_H__ +#define __XDMA_BASE_API_H__ + +#include <linux/types.h> +#include <linux/scatterlist.h> +#include <linux/interrupt.h> + +/* + * functions exported by the xdma driver + */ + +typedef struct { + u64 write_submitted; + u64 write_completed; + u64 read_requested; + u64 read_completed; + u64 restart; + u64 open; + u64 close; + u64 msix_trigger; +} xdma_statistics; + +/* + * This struct should be constantly updated by XMDA using u64_stats_* APIs + * The front end will read the structure without locking (That's why updating atomically is a must) + * every time it prints the statistics. + */ +//static XDMA_Statistics stats; + +/* + * xdma_device_open - read the pci bars and configure the fpga + * should be called from probe() + * NOTE: + * user interrupt will not enabled until xdma_user_isr_enable() + * is called + * @pdev: ptr to pci_dev + * @mod_name: the module name to be used for request_irq + * @user_max: max # of user/event (interrupts) to be configured + * @channel_max: max # of c2h and h2c channels to be configured + * NOTE: if the user/channel provisioned is less than the max specified, + * libxdma will update the user_max/channel_max + * returns + * a opaque handle (for libxdma to identify the device) + * NULL, in case of error + */ +void *xdma_device_open(const char *mod_name, struct pci_dev *pdev, + int *user_max, int *h2c_channel_max, int *c2h_channel_max); + +/* + * xdma_device_close - prepare fpga for removal: disable all interrupts (users + * and xdma) and release all resources + * should called from remove() + * @pdev: ptr to struct pci_dev + * @tuples: from xdma_device_open() + */ +void xdma_device_close(struct pci_dev *pdev, void *dev_handle); + +/* + * xdma_device_restart - restart the fpga + * @pdev: ptr to struct pci_dev + * TODO: + * may need more refining on the parameter list + * return < 0 in case of error + * TODO: exact error code will be defined later + */ +int xdma_device_restart(struct pci_dev *pdev, void *dev_handle); + +/* + * xdma_user_isr_register - register a user ISR handler + * It is expected that the xdma will register the ISR, and for the user + * interrupt, it will call the corresponding handle if it is registered and + * enabled. + * + * @pdev: ptr to the the pci_dev struct + * @mask: bitmask of user interrupts (0 ~ 15)to be registered + * bit 0: user interrupt 0 + * ... + * bit 15: user interrupt 15 + * any bit above bit 15 will be ignored. + * @handler: the correspoinding handler + * a NULL handler will be treated as de-registeration + * @name: to be passed to the handler, ignored if handler is NULL` + * @dev: to be passed to the handler, ignored if handler is NULL` + * return < 0 in case of error + * TODO: exact error code will be defined later + */ +int xdma_user_isr_register(void *dev_hndl, unsigned int mask, + irq_handler_t handler, void *dev); + +/* + * xdma_user_isr_enable/disable - enable or disable user interrupt + * @pdev: ptr to the the pci_dev struct + * @mask: bitmask of user interrupts (0 ~ 15)to be registered + * return < 0 in case of error + * TODO: exact error code will be defined later + */ +int xdma_user_isr_enable(void *dev_hndl, unsigned int mask); +int xdma_user_isr_disable(void *dev_hndl, unsigned int mask); + +/* + * xdma_xfer_submit - submit data for dma operation (for both read and write) + * This is a blocking call + * @channel: channle number (< channel_max) + * == channel_max means libxdma can pick any channel available:q + + * @dir: DMA_FROM/TO_DEVICE + * @offset: offset into the DDR/BRAM memory to read from or write to + * @sg_tbl: the scatter-gather list of data buffers + * @timeout: timeout in mili-seconds, *currently ignored + * return # of bytes transfered or + * < 0 in case of error + * TODO: exact error code will be defined later + */ +ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, + struct sg_table *sgt, bool dma_mapped, int timeout_ms); + + +#endif diff --git a/sources/xdma_driver/libxdma/Makefile b/sources/xdma_driver/libxdma/Makefile new file mode 100644 index 0000000..1003f0f --- /dev/null +++ b/sources/xdma_driver/libxdma/Makefile @@ -0,0 +1,25 @@ +SHELL = /bin/bash + +topdir := $(shell cd $(src)/.. && pwd) + +TARGET_MODULE:=libxdma + +EXTRA_CFLAGS := -I$(topdir)/include +EXTRA_CFLAGS += -D__LIBXDMA_MOD__ + +ifneq ($(KERNELRELEASE),) + obj-m := $(TARGET_MODULE).o +# $(TARGET_MODULE)-objs := libxdma.o +else + BUILDSYSTEM_DIR:=/lib/modules/$(shell uname -r)/build + PWD:=$(shell pwd) +all : + $(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) modules + +clean: + $(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) clean + +install: all + $(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) modules_install + +endif diff --git a/sources/xdma_driver/libxdma/libxdma.c b/sources/xdma_driver/libxdma/libxdma.c new file mode 100644 index 0000000..b9884d7 --- /dev/null +++ b/sources/xdma_driver/libxdma/libxdma.c @@ -0,0 +1,4442 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/string.h> +#include <linux/mm.h> +#include <linux/errno.h> +#include <linux/sched.h> +#include <linux/vmalloc.h> + +#include "libxdma.h" +#include "libxdma_api.h" +#include "cdev_sgdma.h" + +/* SECTION: Module licensing */ + +#ifdef __LIBXDMA_MOD__ +#include "version.h" +#define DRV_MODULE_NAME "libxdma" +#define DRV_MODULE_DESC "Xilinx XDMA Base Driver" +#define DRV_MODULE_RELDATE "Feb. 2018" + +static char version[] = + DRV_MODULE_DESC " " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n"; + +MODULE_AUTHOR("Xilinx, Inc."); +MODULE_DESCRIPTION(DRV_MODULE_DESC); +MODULE_VERSION(DRV_MODULE_VERSION); +MODULE_LICENSE("Dual BSD/GPL"); +#endif + +/* Module Parameters */ +static unsigned int poll_mode; +module_param(poll_mode, uint, 0644); +MODULE_PARM_DESC(poll_mode, "Set 1 for hw polling, default is 0 (interrupts)"); + +static unsigned int interrupt_mode; +module_param(interrupt_mode, uint, 0644); +MODULE_PARM_DESC(interrupt_mode, "0 - MSI-x , 1 - MSI, 2 - Legacy"); + +static unsigned int enable_credit_mp; +module_param(enable_credit_mp, uint, 0644); +MODULE_PARM_DESC(enable_credit_mp, "Set 1 to enable creidt feature, default is 0 (no credit control)"); + +unsigned int desc_blen_max = XDMA_DESC_BLEN_MAX; +module_param(desc_blen_max, uint, 0644); +MODULE_PARM_DESC(desc_blen_max, "per descriptor max. buffer length, default is (1 << 28) - 1"); + +/* + * xdma device management + * maintains a list of the xdma devices + */ +static LIST_HEAD(xdev_list); +static DEFINE_MUTEX(xdev_mutex); + +static LIST_HEAD(xdev_rcu_list); +static DEFINE_SPINLOCK(xdev_rcu_lock); + +#ifndef list_last_entry +#define list_last_entry(ptr, type, member) \ + list_entry((ptr)->prev, type, member) +#endif + +static inline void xdev_list_add(struct xdma_dev *xdev) +{ + mutex_lock(&xdev_mutex); + if (list_empty(&xdev_list)) + xdev->idx = 0; + else { + struct xdma_dev *last; + + last = list_last_entry(&xdev_list, struct xdma_dev, list_head); + xdev->idx = last->idx + 1; + } + list_add_tail(&xdev->list_head, &xdev_list); + mutex_unlock(&xdev_mutex); + + dbg_init("dev %s, xdev 0x%p, xdma idx %d.\n", + dev_name(&xdev->pdev->dev), xdev, xdev->idx); + + spin_lock(&xdev_rcu_lock); + list_add_tail_rcu(&xdev->rcu_node, &xdev_rcu_list); + spin_unlock(&xdev_rcu_lock); +} + +#undef list_last_entry + +static inline void xdev_list_remove(struct xdma_dev *xdev) +{ + mutex_lock(&xdev_mutex); + list_del(&xdev->list_head); + mutex_unlock(&xdev_mutex); + + spin_lock(&xdev_rcu_lock); + list_del_rcu(&xdev->rcu_node); + spin_unlock(&xdev_rcu_lock); + synchronize_rcu(); +} + +struct xdma_dev *xdev_find_by_pdev(struct pci_dev *pdev) +{ + struct xdma_dev *xdev, *tmp; + + mutex_lock(&xdev_mutex); + list_for_each_entry_safe(xdev, tmp, &xdev_list, list_head) { + if (xdev->pdev == pdev) { + mutex_unlock(&xdev_mutex); + return xdev; + } + } + mutex_unlock(&xdev_mutex); + return NULL; +} +EXPORT_SYMBOL_GPL(xdev_find_by_pdev); + +static inline int debug_check_dev_hndl(const char *fname, struct pci_dev *pdev, + void *hndl) +{ + struct xdma_dev *xdev; + + if (!pdev) + return -EINVAL; + + xdev = xdev_find_by_pdev(pdev); + if (!xdev) { + pr_info("%s pdev 0x%p, hndl 0x%p, NO match found!\n", + fname, pdev, hndl); + return -EINVAL; + } + if (xdev != hndl) { + pr_err("%s pdev 0x%p, hndl 0x%p != 0x%p!\n", + fname, pdev, hndl, xdev); + return -EINVAL; + } + + return 0; +} + +#ifdef __LIBXDMA_DEBUG__ +/* SECTION: Function definitions */ +inline void __write_register(const char *fn, u32 value, void *iomem, unsigned long off) +{ + pr_err("%s: w reg 0x%lx(0x%p), 0x%x.\n", fn, off, iomem, value); + iowrite32(value, iomem); +} +#define write_register(v,mem,off) __write_register(__func__, v, mem, off) +#else +#define write_register(v,mem,off) iowrite32(v, mem) +#endif + +inline u32 read_register(void *iomem) +{ + return ioread32(iomem); +} + +static inline u32 build_u32(u32 hi, u32 lo) +{ + return ((hi & 0xFFFFUL) << 16) | (lo & 0xFFFFUL); +} + +static inline u64 build_u64(u64 hi, u64 lo) +{ + return ((hi & 0xFFFFFFFULL) << 32) | (lo & 0xFFFFFFFFULL); +} + +static void check_nonzero_interrupt_status(struct xdma_dev *xdev) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + u32 w; + + w = read_register(®->user_int_enable); + if (w) + pr_info("%s xdma%d user_int_enable = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + + w = read_register(®->channel_int_enable); + if (w) + pr_info("%s xdma%d channel_int_enable = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + + w = read_register(®->user_int_request); + if (w) + pr_info("%s xdma%d user_int_request = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + w = read_register(®->channel_int_request); + if (w) + pr_info("%s xdma%d channel_int_request = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + + w = read_register(®->user_int_pending); + if (w) + pr_info("%s xdma%d user_int_pending = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + w = read_register(®->channel_int_pending); + if (w) + pr_info("%s xdma%d channel_int_pending = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); +} + +/* channel_interrupts_enable -- Enable interrupts we are interested in */ +static void channel_interrupts_enable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->channel_int_enable_w1s, XDMA_OFS_INT_CTRL); +} + +/* channel_interrupts_disable -- Disable interrupts we not interested in */ +static void channel_interrupts_disable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->channel_int_enable_w1c, XDMA_OFS_INT_CTRL); +} + +/* user_interrupts_enable -- Enable interrupts we are interested in */ +static void user_interrupts_enable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->user_int_enable_w1s, XDMA_OFS_INT_CTRL); +} + +/* user_interrupts_disable -- Disable interrupts we not interested in */ +static void user_interrupts_disable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->user_int_enable_w1c, XDMA_OFS_INT_CTRL); +} + +/* read_interrupts -- Print the interrupt controller status */ +static u32 read_interrupts(struct xdma_dev *xdev) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + u32 lo; + u32 hi; + + /* extra debugging; inspect complete engine set of registers */ + hi = read_register(®->user_int_request); + dbg_io("ioread32(0x%p) returned 0x%08x (user_int_request).\n", + ®->user_int_request, hi); + lo = read_register(®->channel_int_request); + dbg_io("ioread32(0x%p) returned 0x%08x (channel_int_request)\n", + ®->channel_int_request, lo); + + /* return interrupts: user in upper 16-bits, channel in lower 16-bits */ + return build_u32(hi, lo); +} + +void enable_perf(struct xdma_engine *engine) +{ + u32 w; + + w = XDMA_PERF_CLEAR; + write_register(w, &engine->regs->perf_ctrl, + (unsigned long)(&engine->regs->perf_ctrl) - + (unsigned long)(&engine->regs)); + read_register(&engine->regs->identifier); + w = XDMA_PERF_AUTO | XDMA_PERF_RUN; + write_register(w, &engine->regs->perf_ctrl, + (unsigned long)(&engine->regs->perf_ctrl) - + (unsigned long)(&engine->regs)); + read_register(&engine->regs->identifier); + + dbg_perf("IOCTL_XDMA_PERF_START\n"); + +} +EXPORT_SYMBOL_GPL(enable_perf); + +void get_perf_stats(struct xdma_engine *engine) +{ + u32 hi; + u32 lo; + + BUG_ON(!engine); + + if (!engine->xdma_perf) { + pr_info("%s perf struct not set up.\n", engine->name); + return; + } + + hi = 0; + lo = read_register(&engine->regs->completed_desc_count); + engine->xdma_perf->iterations = build_u64(hi, lo); + + hi = read_register(&engine->regs->perf_cyc_hi); + lo = read_register(&engine->regs->perf_cyc_lo); + + engine->xdma_perf->clock_cycle_count = build_u64(hi, lo); + + hi = read_register(&engine->regs->perf_dat_hi); + lo = read_register(&engine->regs->perf_dat_lo); + engine->xdma_perf->data_cycle_count = build_u64(hi, lo); + + hi = read_register(&engine->regs->perf_pnd_hi); + lo = read_register(&engine->regs->perf_pnd_lo); + engine->xdma_perf->pending_count = build_u64(hi, lo); +} +EXPORT_SYMBOL_GPL(get_perf_stats); + +static void engine_reg_dump(struct xdma_engine *engine) +{ + u32 w; + + BUG_ON(!engine); + + w = read_register(&engine->regs->identifier); + pr_info("%s: ioread32(0x%p) = 0x%08x (id).\n", + engine->name, &engine->regs->identifier, w); + w &= BLOCK_ID_MASK; + if (w != BLOCK_ID_HEAD) { + pr_info("%s: engine id missing, 0x%08x exp. & 0x%x = 0x%x\n", + engine->name, w, BLOCK_ID_MASK, BLOCK_ID_HEAD); + return; + } + /* extra debugging; inspect complete engine set of registers */ + w = read_register(&engine->regs->status); + pr_info("%s: ioread32(0x%p) = 0x%08x (status).\n", + engine->name, &engine->regs->status, w); + w = read_register(&engine->regs->control); + pr_info("%s: ioread32(0x%p) = 0x%08x (control)\n", + engine->name, &engine->regs->control, w); + w = read_register(&engine->sgdma_regs->first_desc_lo); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_lo)\n", + engine->name, &engine->sgdma_regs->first_desc_lo, w); + w = read_register(&engine->sgdma_regs->first_desc_hi); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_hi)\n", + engine->name, &engine->sgdma_regs->first_desc_hi, w); + w = read_register(&engine->sgdma_regs->first_desc_adjacent); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_adjacent).\n", + engine->name, &engine->sgdma_regs->first_desc_adjacent, w); + w = read_register(&engine->regs->completed_desc_count); + pr_info("%s: ioread32(0x%p) = 0x%08x (completed_desc_count).\n", + engine->name, &engine->regs->completed_desc_count, w); + w = read_register(&engine->regs->interrupt_enable_mask); + pr_info("%s: ioread32(0x%p) = 0x%08x (interrupt_enable_mask)\n", + engine->name, &engine->regs->interrupt_enable_mask, w); +} + +/** + * engine_status_read() - read status of SG DMA engine (optionally reset) + * + * Stores status in engine->status. + * + * @return -1 on failure, status register otherwise + */ +static void engine_status_dump(struct xdma_engine *engine) +{ + u32 v = engine->status; + char buffer[256]; + char *buf = buffer; + int len = 0; + + len = sprintf(buf, "SG engine %s status: 0x%08x: ", engine->name, v); + + if ((v & XDMA_STAT_BUSY)) + len += sprintf(buf + len, "BUSY,"); + if ((v & XDMA_STAT_DESC_STOPPED)) + len += sprintf(buf + len, "DESC_STOPPED,"); + if ((v & XDMA_STAT_DESC_COMPLETED)) + len += sprintf(buf + len, "DESC_COMPL,"); + + /* common H2C & C2H */ + if ((v & XDMA_STAT_COMMON_ERR_MASK)) { + if ((v & XDMA_STAT_ALIGN_MISMATCH)) + len += sprintf(buf + len, "ALIGN_MISMATCH "); + if ((v & XDMA_STAT_MAGIC_STOPPED)) + len += sprintf(buf + len, "MAGIC_STOPPED "); + if ((v & XDMA_STAT_INVALID_LEN)) + len += sprintf(buf + len, "INVLIAD_LEN "); + if ((v & XDMA_STAT_IDLE_STOPPED)) + len += sprintf(buf + len, "IDLE_STOPPED "); + buf[len - 1] = ','; + } + + if ((engine->dir == DMA_TO_DEVICE)) { + /* H2C only */ + if ((v & XDMA_STAT_H2C_R_ERR_MASK)) { + len += sprintf(buf + len, "R:"); + if ((v & XDMA_STAT_H2C_R_UNSUPP_REQ)) + len += sprintf(buf + len, "UNSUPP_REQ "); + if ((v & XDMA_STAT_H2C_R_COMPL_ABORT)) + len += sprintf(buf + len, "COMPL_ABORT "); + if ((v & XDMA_STAT_H2C_R_PARITY_ERR)) + len += sprintf(buf + len, "PARITY "); + if ((v & XDMA_STAT_H2C_R_HEADER_EP)) + len += sprintf(buf + len, "HEADER_EP "); + if ((v & XDMA_STAT_H2C_R_UNEXP_COMPL)) + len += sprintf(buf + len, "UNEXP_COMPL "); + buf[len - 1] = ','; + } + + if ((v & XDMA_STAT_H2C_W_ERR_MASK)) { + len += sprintf(buf + len, "W:"); + if ((v & XDMA_STAT_H2C_W_DECODE_ERR)) + len += sprintf(buf + len, "DECODE_ERR "); + if ((v & XDMA_STAT_H2C_W_SLAVE_ERR)) + len += sprintf(buf + len, "SLAVE_ERR "); + buf[len - 1] = ','; + } + + } else { + /* C2H only */ + if ((v & XDMA_STAT_C2H_R_ERR_MASK)) { + len += sprintf(buf + len, "R:"); + if ((v & XDMA_STAT_C2H_R_DECODE_ERR)) + len += sprintf(buf + len, "DECODE_ERR "); + if ((v & XDMA_STAT_C2H_R_SLAVE_ERR)) + len += sprintf(buf + len, "SLAVE_ERR "); + buf[len - 1] = ','; + } + } + + /* common H2C & C2H */ + if ((v & XDMA_STAT_DESC_ERR_MASK)) { + len += sprintf(buf + len, "DESC_ERR:"); + if ((v & XDMA_STAT_DESC_UNSUPP_REQ)) + len += sprintf(buf + len, "UNSUPP_REQ "); + if ((v & XDMA_STAT_DESC_COMPL_ABORT)) + len += sprintf(buf + len, "COMPL_ABORT "); + if ((v & XDMA_STAT_DESC_PARITY_ERR)) + len += sprintf(buf + len, "PARITY "); + if ((v & XDMA_STAT_DESC_HEADER_EP)) + len += sprintf(buf + len, "HEADER_EP "); + if ((v & XDMA_STAT_DESC_UNEXP_COMPL)) + len += sprintf(buf + len, "UNEXP_COMPL "); + buf[len - 1] = ','; + } + + buf[len - 1] = '\0'; + pr_info("%s\n", buffer); +} + +static u32 engine_status_read(struct xdma_engine *engine, bool clear, bool dump) +{ + u32 value; + + BUG_ON(!engine); + + if (dump) + engine_reg_dump(engine); + + /* read status register */ + if (clear) + value = engine->status = + read_register(&engine->regs->status_rc); + else + value = engine->status = read_register(&engine->regs->status); + + if (dump) + engine_status_dump(engine); + + return value; +} + +/** + * xdma_engine_stop() - stop an SG DMA engine + * + */ +static void xdma_engine_stop(struct xdma_engine *engine) +{ + u32 w; + + BUG_ON(!engine); + dbg_tfr("xdma_engine_stop(engine=%p)\n", engine); + + w = 0; + w |= (u32)XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + w |= (u32)XDMA_CTRL_IE_MAGIC_STOPPED; + w |= (u32)XDMA_CTRL_IE_READ_ERROR; + w |= (u32)XDMA_CTRL_IE_DESC_ERROR; + + if (poll_mode) { + w |= (u32) XDMA_CTRL_POLL_MODE_WB; + } else { + w |= (u32)XDMA_CTRL_IE_DESC_STOPPED; + w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED; + + /* Disable IDLE STOPPED for MM */ + if ((engine->streaming && (engine->dir == DMA_FROM_DEVICE)) || + (engine->xdma_perf)) + w |= (u32)XDMA_CTRL_IE_IDLE_STOPPED; + } + + dbg_tfr("Stopping SG DMA %s engine; writing 0x%08x to 0x%p.\n", + engine->name, w, (u32 *)&engine->regs->control); + write_register(w, &engine->regs->control, + (unsigned long)(&engine->regs->control) - + (unsigned long)(&engine->regs)); + /* dummy read of status register to flush all previous writes */ + dbg_tfr("xdma_engine_stop(%s) done\n", engine->name); +} + +static void engine_start_mode_config(struct xdma_engine *engine) +{ + u32 w; + + BUG_ON(!engine); + + /* If a perf test is running, enable the engine interrupts */ + if (engine->xdma_perf) { + w = XDMA_CTRL_IE_DESC_STOPPED; + w |= XDMA_CTRL_IE_DESC_COMPLETED; + w |= XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + w |= XDMA_CTRL_IE_MAGIC_STOPPED; + w |= XDMA_CTRL_IE_IDLE_STOPPED; + w |= XDMA_CTRL_IE_READ_ERROR; + w |= XDMA_CTRL_IE_DESC_ERROR; + + write_register(w, &engine->regs->interrupt_enable_mask, + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); + } + + /* write control register of SG DMA engine */ + w = (u32)XDMA_CTRL_RUN_STOP; + w |= (u32)XDMA_CTRL_IE_READ_ERROR; + w |= (u32)XDMA_CTRL_IE_DESC_ERROR; + w |= (u32)XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + w |= (u32)XDMA_CTRL_IE_MAGIC_STOPPED; + + if (poll_mode) { + w |= (u32)XDMA_CTRL_POLL_MODE_WB; + } else { + w |= (u32)XDMA_CTRL_IE_DESC_STOPPED; + w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED; + + if ((engine->streaming && (engine->dir == DMA_FROM_DEVICE)) || + (engine->xdma_perf)) + w |= (u32)XDMA_CTRL_IE_IDLE_STOPPED; + + /* set non-incremental addressing mode */ + if (engine->non_incr_addr) + w |= (u32)XDMA_CTRL_NON_INCR_ADDR; + } + + dbg_tfr("iowrite32(0x%08x to 0x%p) (control)\n", w, + (void *)&engine->regs->control); + /* start the engine */ + write_register(w, &engine->regs->control, + (unsigned long)(&engine->regs->control) - + (unsigned long)(&engine->regs)); + + /* dummy read of status register to flush all previous writes */ + w = read_register(&engine->regs->status); + dbg_tfr("ioread32(0x%p) = 0x%08x (dummy read flushes writes).\n", + &engine->regs->status, w); +} + +/** + * engine_start() - start an idle engine with its first transfer on queue + * + * The engine will run and process all transfers that are queued using + * transfer_queue() and thus have their descriptor lists chained. + * + * During the run, new transfers will be processed if transfer_queue() has + * chained the descriptors before the hardware fetches the last descriptor. + * A transfer that was chained too late will invoke a new run of the engine + * initiated from the engine_service() routine. + * + * The engine must be idle and at least one transfer must be queued. + * This function does not take locks; the engine spinlock must already be + * taken. + * + */ +static struct xdma_transfer *engine_start(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer; + u32 w; + int extra_adj = 0; + + /* engine must be idle */ + BUG_ON(engine->running); + /* engine transfer queue must not be empty */ + BUG_ON(list_empty(&engine->transfer_list)); + /* inspect first transfer queued on the engine */ + transfer = list_entry(engine->transfer_list.next, struct xdma_transfer, + entry); + BUG_ON(!transfer); + + /* engine is no longer shutdown */ + engine->shutdown = ENGINE_SHUTDOWN_NONE; + + dbg_tfr("engine_start(%s): transfer=0x%p.\n", engine->name, transfer); + + /* initialize number of descriptors of dequeued transfers */ + engine->desc_dequeued = 0; + + /* write lower 32-bit of bus address of transfer first descriptor */ + w = cpu_to_le32(PCI_DMA_L(transfer->desc_bus)); + dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_lo)\n", w, + (void *)&engine->sgdma_regs->first_desc_lo); + write_register(w, &engine->sgdma_regs->first_desc_lo, + (unsigned long)(&engine->sgdma_regs->first_desc_lo) - + (unsigned long)(&engine->sgdma_regs)); + /* write upper 32-bit of bus address of transfer first descriptor */ + w = cpu_to_le32(PCI_DMA_H(transfer->desc_bus)); + dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_hi)\n", w, + (void *)&engine->sgdma_regs->first_desc_hi); + write_register(w, &engine->sgdma_regs->first_desc_hi, + (unsigned long)(&engine->sgdma_regs->first_desc_hi) - + (unsigned long)(&engine->sgdma_regs)); + + if (transfer->desc_adjacent > 0) { + extra_adj = transfer->desc_adjacent - 1; + if (extra_adj > MAX_EXTRA_ADJ) + extra_adj = MAX_EXTRA_ADJ; + } + dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_adjacent)\n", + extra_adj, (void *)&engine->sgdma_regs->first_desc_adjacent); + write_register(extra_adj, &engine->sgdma_regs->first_desc_adjacent, + (unsigned long)(&engine->sgdma_regs->first_desc_adjacent) - + (unsigned long)(&engine->sgdma_regs)); + + dbg_tfr("ioread32(0x%p) (dummy read flushes writes).\n", + &engine->regs->status); + mmiowb(); + + engine_start_mode_config(engine); + + engine_status_read(engine, 0, 0); + + dbg_tfr("%s engine 0x%p now running\n", engine->name, engine); + /* remember the engine is running */ + engine->running = 1; + return transfer; +} + +/** + * engine_service() - service an SG DMA engine + * + * must be called with engine->lock already acquired + * + * @engine pointer to struct xdma_engine + * + */ +static void engine_service_shutdown(struct xdma_engine *engine) +{ + /* if the engine stopped with RUN still asserted, de-assert RUN now */ + dbg_tfr("engine just went idle, resetting RUN_STOP.\n"); + xdma_engine_stop(engine); + engine->running = 0; + + /* awake task on engine's shutdown wait queue */ + wake_up_interruptible(&engine->shutdown_wq); +} + +struct xdma_transfer *engine_transfer_completion(struct xdma_engine *engine, + struct xdma_transfer *transfer) +{ + BUG_ON(!engine); + + if (unlikely(!transfer)) { + pr_info("%s: xfer empty.\n", engine->name); + return NULL; + } + + /* synchronous I/O? */ + /* awake task on transfer's wait queue */ + wake_up_interruptible(&transfer->wq); + + return transfer; +} + +struct xdma_transfer *engine_service_transfer_list(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 *pdesc_completed) +{ + BUG_ON(!engine); + BUG_ON(!pdesc_completed); + + if (unlikely(!transfer)) { + pr_info("%s xfer empty, pdesc completed %u.\n", + engine->name, *pdesc_completed); + return NULL; + } + + /* + * iterate over all the transfers completed by the engine, + * except for the last (i.e. use > instead of >=). + */ + while (transfer && (!transfer->cyclic) && + (*pdesc_completed > transfer->desc_num)) { + /* remove this transfer from pdesc_completed */ + *pdesc_completed -= transfer->desc_num; + dbg_tfr("%s engine completed non-cyclic xfer 0x%p (%d desc)\n", + engine->name, transfer, transfer->desc_num); + /* remove completed transfer from list */ + list_del(engine->transfer_list.next); + /* add to dequeued number of descriptors during this run */ + engine->desc_dequeued += transfer->desc_num; + /* mark transfer as succesfully completed */ + transfer->state = TRANSFER_STATE_COMPLETED; + + /* Complete transfer - sets transfer to NULL if an async + * transfer has completed */ + transfer = engine_transfer_completion(engine, transfer); + + /* if exists, get the next transfer on the list */ + if (!list_empty(&engine->transfer_list)) { + transfer = list_entry(engine->transfer_list.next, + struct xdma_transfer, entry); + dbg_tfr("Non-completed transfer %p\n", transfer); + } else { + /* no further transfers? */ + transfer = NULL; + } + } + + return transfer; +} + +static void engine_err_handle(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 desc_completed) +{ + u32 value; + + /* + * The BUSY bit is expected to be clear now but older HW has a race + * condition which could cause it to be still set. If it's set, re-read + * and check again. If it's still set, log the issue. + */ + if (engine->status & XDMA_STAT_BUSY) { + value = read_register(&engine->regs->status); + if ((value & XDMA_STAT_BUSY) && printk_ratelimit()) + pr_info("%s has errors but is still BUSY\n", + engine->name); + } + + if (printk_ratelimit()) { + pr_info("%s, s 0x%x, aborted xfer 0x%p, cmpl %d/%d\n", + engine->name, engine->status, transfer, desc_completed, + transfer->desc_num); + } + + /* mark transfer as failed */ + transfer->state = TRANSFER_STATE_FAILED; + xdma_engine_stop(engine); +} + +struct xdma_transfer *engine_service_final_transfer(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 *pdesc_completed) +{ + BUG_ON(!engine); + BUG_ON(!pdesc_completed); + + /* inspect the current transfer */ + if (unlikely(!transfer)) { + pr_info("%s xfer empty, pdesc completed %u.\n", + engine->name, *pdesc_completed); + return NULL; + } else { + if (((engine->dir == DMA_FROM_DEVICE) && + (engine->status & XDMA_STAT_C2H_ERR_MASK)) || + ((engine->dir == DMA_TO_DEVICE) && + (engine->status & XDMA_STAT_H2C_ERR_MASK))) { + pr_info("engine %s, status error 0x%x.\n", + engine->name, engine->status); + engine_status_dump(engine); + engine_err_handle(engine, transfer, *pdesc_completed); + goto transfer_del; + } + + if (engine->status & XDMA_STAT_BUSY) + pr_debug("engine %s is unexpectedly busy - ignoring\n", + engine->name); + + /* the engine stopped on current transfer? */ + if (*pdesc_completed < transfer->desc_num) { + transfer->state = TRANSFER_STATE_FAILED; + pr_info("%s, xfer 0x%p, stopped half-way, %d/%d.\n", + engine->name, transfer, *pdesc_completed, + transfer->desc_num); + } else { + dbg_tfr("engine %s completed transfer\n", engine->name); + dbg_tfr("Completed transfer ID = 0x%p\n", transfer); + dbg_tfr("*pdesc_completed=%d, transfer->desc_num=%d", + *pdesc_completed, transfer->desc_num); + + if (!transfer->cyclic) { + /* + * if the engine stopped on this transfer, + * it should be the last + */ + WARN_ON(*pdesc_completed > transfer->desc_num); + } + /* mark transfer as succesfully completed */ + transfer->state = TRANSFER_STATE_COMPLETED; + } + +transfer_del: + /* remove completed transfer from list */ + list_del(engine->transfer_list.next); + /* add to dequeued number of descriptors during this run */ + engine->desc_dequeued += transfer->desc_num; + + /* + * Complete transfer - sets transfer to NULL if an asynchronous + * transfer has completed + */ + transfer = engine_transfer_completion(engine, transfer); + } + + return transfer; +} + +static void engine_service_perf(struct xdma_engine *engine, u32 desc_completed) +{ + BUG_ON(!engine); + + /* performance measurement is running? */ + if (engine->xdma_perf) { + /* a descriptor was completed? */ + if (engine->status & XDMA_STAT_DESC_COMPLETED) { + engine->xdma_perf->iterations = desc_completed; + dbg_perf("transfer->xdma_perf->iterations=%d\n", + engine->xdma_perf->iterations); + } + + /* a descriptor stopped the engine? */ + if (engine->status & XDMA_STAT_DESC_STOPPED) { + engine->xdma_perf->stopped = 1; + /* + * wake any XDMA_PERF_IOCTL_STOP waiting for + * the performance run to finish + */ + wake_up_interruptible(&engine->xdma_perf_wq); + dbg_perf("transfer->xdma_perf stopped\n"); + } + } +} + +static void engine_transfer_dequeue(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer; + + BUG_ON(!engine); + + /* pick first transfer on the queue (was submitted to the engine) */ + transfer = list_entry(engine->transfer_list.next, struct xdma_transfer, + entry); + if (!transfer || transfer != &engine->cyclic_req->xfer) { + pr_info("%s, xfer 0x%p != 0x%p.\n", + engine->name, transfer, &engine->cyclic_req->xfer); + return; + } + dbg_tfr("%s engine completed cyclic transfer 0x%p (%d desc).\n", + engine->name, transfer, transfer->desc_num); + /* remove completed transfer from list */ + list_del(engine->transfer_list.next); +} + +static int engine_ring_process(struct xdma_engine *engine) +{ + struct xdma_result *result; + int start; + int eop_count = 0; + + BUG_ON(!engine); + result = engine->cyclic_result; + BUG_ON(!result); + + /* where we start receiving in the ring buffer */ + start = engine->rx_tail; + + /* iterate through all newly received RX result descriptors */ + dbg_tfr("%s, result %d, 0x%x, len 0x%x.\n", + engine->name, engine->rx_tail, result[engine->rx_tail].status, + result[engine->rx_tail].length); + while (result[engine->rx_tail].status && !engine->rx_overrun) { + /* EOP bit set in result? */ + if (result[engine->rx_tail].status & RX_STATUS_EOP){ + eop_count++; + } + + /* increment tail pointer */ + engine->rx_tail = (engine->rx_tail + 1) % CYCLIC_RX_PAGES_MAX; + + dbg_tfr("%s, head %d, tail %d, 0x%x, len 0x%x.\n", + engine->name, engine->rx_head, engine->rx_tail, + result[engine->rx_tail].status, + result[engine->rx_tail].length); + + /* overrun? */ + if (engine->rx_tail == engine->rx_head) { + dbg_tfr("%s: overrun\n", engine->name); + /* flag to user space that overrun has occurred */ + engine->rx_overrun = 1; + } + } + + return eop_count; +} + +static int engine_service_cyclic_polled(struct xdma_engine *engine) +{ + int eop_count = 0; + int rc = 0; + struct xdma_poll_wb *writeback_data; + u32 sched_limit = 0; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + writeback_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + + while (eop_count == 0) { + if (sched_limit != 0) { + if ((sched_limit % NUM_POLLS_PER_SCHED) == 0) + schedule(); + } + sched_limit++; + + /* Monitor descriptor writeback address for errors */ + if ((writeback_data->completed_desc_count) & WB_ERR_MASK) { + rc = -1; + break; + } + + eop_count = engine_ring_process(engine); + } + + if (eop_count == 0) { + engine_status_read(engine, 1, 0); + if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) + engine_transfer_dequeue(engine); + + engine_service_shutdown(engine); + } + } + + return rc; +} + +static int engine_service_cyclic_interrupt(struct xdma_engine *engine) +{ + int eop_count = 0; + struct xdma_transfer *xfer; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + engine_status_read(engine, 1, 0); + + eop_count = engine_ring_process(engine); + /* + * wake any reader on EOP, as one or more packets are now in + * the RX buffer + */ + xfer = &engine->cyclic_req->xfer; + if(enable_credit_mp){ + if (eop_count > 0) { + //engine->eop_found = 1; + } + wake_up_interruptible(&xfer->wq); + }else{ + if (eop_count > 0) { + /* awake task on transfer's wait queue */ + dbg_tfr("wake_up_interruptible() due to %d EOP's\n", eop_count); + engine->eop_found = 1; + wake_up_interruptible(&xfer->wq); + } + } + + /* engine was running but is no longer busy? */ + if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) + engine_transfer_dequeue(engine); + + engine_service_shutdown(engine); + } + + return 0; +} + +/* must be called with engine->lock already acquired */ +static int engine_service_cyclic(struct xdma_engine *engine) +{ + int rc = 0; + + dbg_tfr("engine_service_cyclic()"); + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + if (poll_mode) + rc = engine_service_cyclic_polled(engine); + else + rc = engine_service_cyclic_interrupt(engine); + + return rc; +} + + +static void engine_service_resume(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer_started; + + BUG_ON(!engine); + + /* engine stopped? */ + if (!engine->running) { + /* in the case of shutdown, let it finish what's in the Q */ + if (!list_empty(&engine->transfer_list)) { + /* (re)start engine */ + transfer_started = engine_start(engine); + pr_info("re-started %s engine with pending xfer 0x%p\n", + engine->name, transfer_started); + /* engine was requested to be shutdown? */ + } else if (engine->shutdown & ENGINE_SHUTDOWN_REQUEST) { + engine->shutdown |= ENGINE_SHUTDOWN_IDLE; + /* awake task on engine's shutdown wait queue */ + wake_up_interruptible(&engine->shutdown_wq); + } else { + dbg_tfr("no pending transfers, %s engine stays idle.\n", + engine->name); + } + } else { + /* engine is still running? */ + if (list_empty(&engine->transfer_list)) { + pr_warn("no queued transfers but %s engine running!\n", + engine->name); + WARN_ON(1); + } + } +} + +/** + * engine_service() - service an SG DMA engine + * + * must be called with engine->lock already acquired + * + * @engine pointer to struct xdma_engine + * + */ +static int engine_service(struct xdma_engine *engine, int desc_writeback) +{ + struct xdma_transfer *transfer = NULL; + u32 desc_count = desc_writeback & WB_COUNT_MASK; + u32 err_flag = desc_writeback & WB_ERR_MASK; + int rv = 0; + struct xdma_poll_wb *wb_data; + + BUG_ON(!engine); + + /* If polling detected an error, signal to the caller */ + if (err_flag) + rv = -1; + + /* Service the engine */ + if (!engine->running) { + dbg_tfr("Engine was not running!!! Clearing status\n"); + engine_status_read(engine, 1, 0); + return 0; + } + + /* + * If called by the ISR or polling detected an error, read and clear + * engine status. For polled mode descriptor completion, this read is + * unnecessary and is skipped to reduce latency + */ + if ((desc_count == 0) || (err_flag != 0)) + engine_status_read(engine, 1, 0); + + /* + * engine was running but is no longer busy, or writeback occurred, + * shut down + */ + if ((engine->running && !(engine->status & XDMA_STAT_BUSY)) || + (desc_count != 0)) + engine_service_shutdown(engine); + + /* + * If called from the ISR, or if an error occurred, the descriptor + * count will be zero. In this scenario, read the descriptor count + * from HW. In polled mode descriptor completion, this read is + * unnecessary and is skipped to reduce latency + */ + if (!desc_count) + desc_count = read_register(&engine->regs->completed_desc_count); + dbg_tfr("desc_count = %d\n", desc_count); + + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) { + /* pick first transfer on queue (was submitted to the engine) */ + transfer = list_entry(engine->transfer_list.next, + struct xdma_transfer, entry); + + dbg_tfr("head of queue transfer 0x%p has %d descriptors\n", + transfer, (int)transfer->desc_num); + + dbg_tfr("Engine completed %d desc, %d not yet dequeued\n", + (int)desc_count, + (int)desc_count - engine->desc_dequeued); + + engine_service_perf(engine, desc_count); + } + + /* account for already dequeued transfers during this engine run */ + desc_count -= engine->desc_dequeued; + + /* Process all but the last transfer */ + transfer = engine_service_transfer_list(engine, transfer, &desc_count); + + /* + * Process final transfer - includes checks of number of descriptors to + * detect faulty completion + */ + transfer = engine_service_final_transfer(engine, transfer, &desc_count); + + /* Before starting engine again, clear the writeback data */ + if (poll_mode) { + wb_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + wb_data->completed_desc_count = 0; + } + + /* Restart the engine following the servicing */ + engine_service_resume(engine); + + return 0; +} + +/* engine_service_work */ +static void engine_service_work(struct work_struct *work) +{ + struct xdma_engine *engine; + unsigned long flags; + + engine = container_of(work, struct xdma_engine, work); + BUG_ON(engine->magic != MAGIC_ENGINE); + + /* lock the engine */ + spin_lock_irqsave(&engine->lock, flags); + + dbg_tfr("engine_service() for %s engine %p\n", + engine->name, engine); + if (engine->cyclic_req) + engine_service_cyclic(engine); + else + engine_service(engine, 0); + + /* re-enable interrupts for this engine */ + if (engine->xdev->msix_enabled){ + write_register(engine->interrupt_enable_mask_value, + &engine->regs->interrupt_enable_mask_w1s, + (unsigned long)(&engine->regs->interrupt_enable_mask_w1s) - + (unsigned long)(&engine->regs)); + } else + channel_interrupts_enable(engine->xdev, engine->irq_bitmask); + + /* unlock the engine */ + spin_unlock_irqrestore(&engine->lock, flags); +} + +static u32 engine_service_wb_monitor(struct xdma_engine *engine, + u32 expected_wb) +{ + struct xdma_poll_wb *wb_data; + u32 desc_wb = 0; + u32 sched_limit = 0; + unsigned long timeout; + + BUG_ON(!engine); + wb_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + + /* + * Poll the writeback location for the expected number of + * descriptors / error events This loop is skipped for cyclic mode, + * where the expected_desc_count passed in is zero, since it cannot be + * determined before the function is called + */ + + timeout = jiffies + (POLL_TIMEOUT_SECONDS * HZ); + while (expected_wb != 0) { + desc_wb = wb_data->completed_desc_count; + + if (desc_wb & WB_ERR_MASK) + break; + else if (desc_wb == expected_wb) + break; + + /* RTO - prevent system from hanging in polled mode */ + if (time_after(jiffies, timeout)) { + dbg_tfr("Polling timeout occurred"); + dbg_tfr("desc_wb = 0x%08x, expected 0x%08x\n", desc_wb, + expected_wb); + if ((desc_wb & WB_COUNT_MASK) > expected_wb) + desc_wb = expected_wb | WB_ERR_MASK; + + break; + } + + /* + * Define NUM_POLLS_PER_SCHED to limit how much time is spent + * in the scheduler + */ + + if (sched_limit != 0) { + if ((sched_limit % NUM_POLLS_PER_SCHED) == 0) + schedule(); + } + sched_limit++; + } + + return desc_wb; +} + +static int engine_service_poll(struct xdma_engine *engine, + u32 expected_desc_count) +{ + struct xdma_poll_wb *writeback_data; + u32 desc_wb = 0; + unsigned long flags; + int rv = 0; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + writeback_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + + if ((expected_desc_count & WB_COUNT_MASK) != expected_desc_count) { + dbg_tfr("Queued descriptor count is larger than supported\n"); + return -1; + } + + /* + * Poll the writeback location for the expected number of + * descriptors / error events This loop is skipped for cyclic mode, + * where the expected_desc_count passed in is zero, since it cannot be + * determined before the function is called + */ + + desc_wb = engine_service_wb_monitor(engine, expected_desc_count); + + spin_lock_irqsave(&engine->lock, flags); + dbg_tfr("%s service.\n", engine->name); + if (engine->cyclic_req) { + rv = engine_service_cyclic(engine); + } else { + rv = engine_service(engine, desc_wb); + } + spin_unlock_irqrestore(&engine->lock, flags); + + return rv; +} + +static irqreturn_t user_irq_service(int irq, struct xdma_user_irq *user_irq) +{ + unsigned long flags; + + BUG_ON(!user_irq); + + if (user_irq->handler) + return user_irq->handler(user_irq->user_idx, user_irq->dev); + + spin_lock_irqsave(&(user_irq->events_lock), flags); + if (!user_irq->events_irq) { + user_irq->events_irq = 1; + wake_up_interruptible(&(user_irq->events_wq)); + } + spin_unlock_irqrestore(&(user_irq->events_lock), flags); + + return IRQ_HANDLED; +} + +/* + * xdma_isr() - Interrupt handler + * + * @dev_id pointer to xdma_dev + */ +static irqreturn_t xdma_isr(int irq, void *dev_id) +{ + u32 ch_irq; + u32 user_irq; + u32 mask; + struct xdma_dev *xdev; + struct interrupt_regs *irq_regs; + + dbg_irq("(irq=%d, dev 0x%p) <<<< ISR.\n", irq, dev_id); + BUG_ON(!dev_id); + xdev = (struct xdma_dev *)dev_id; + + if (!xdev) { + WARN_ON(!xdev); + dbg_irq("xdma_isr(irq=%d) xdev=%p ??\n", irq, xdev); + return IRQ_NONE; + } + + irq_regs = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + + /* read channel interrupt requests */ + ch_irq = read_register(&irq_regs->channel_int_request); + dbg_irq("ch_irq = 0x%08x\n", ch_irq); + + /* + * disable all interrupts that fired; these are re-enabled individually + * after the causing module has been fully serviced. + */ + if (ch_irq) + channel_interrupts_disable(xdev, ch_irq); + + /* read user interrupts - this read also flushes the above write */ + user_irq = read_register(&irq_regs->user_int_request); + dbg_irq("user_irq = 0x%08x\n", user_irq); + + if (user_irq) { + int user = 0; + u32 mask = 1; + int max = xdev->h2c_channel_max; + + for (; user < max && user_irq; user++, mask <<= 1) { + if (user_irq & mask) { + user_irq &= ~mask; + user_irq_service(irq, &xdev->user_irq[user]); + } + } + } + + mask = ch_irq & xdev->mask_irq_h2c; + if (mask) { + int channel = 0; + int max = xdev->h2c_channel_max; + + /* iterate over H2C (PCIe read) */ + for (channel = 0; channel < max && mask; channel++) { + struct xdma_engine *engine = &xdev->engine_h2c[channel]; + + /* engine present and its interrupt fired? */ + if((engine->irq_bitmask & mask) && + (engine->magic == MAGIC_ENGINE)) { + mask &= ~engine->irq_bitmask; + dbg_tfr("schedule_work, %s.\n", engine->name); + schedule_work(&engine->work); + } + } + } + + mask = ch_irq & xdev->mask_irq_c2h; + if (mask) { + int channel = 0; + int max = xdev->c2h_channel_max; + + /* iterate over C2H (PCIe write) */ + for (channel = 0; channel < max && mask; channel++) { + struct xdma_engine *engine = &xdev->engine_c2h[channel]; + + /* engine present and its interrupt fired? */ + if((engine->irq_bitmask & mask) && + (engine->magic == MAGIC_ENGINE)) { + mask &= ~engine->irq_bitmask; + dbg_tfr("schedule_work, %s.\n", engine->name); + schedule_work(&engine->work); + } + } + } + + xdev->irq_count++; + return IRQ_HANDLED; +} + +/* + * xdma_user_irq() - Interrupt handler for user interrupts in MSI-X mode + * + * @dev_id pointer to xdma_dev + */ +static irqreturn_t xdma_user_irq(int irq, void *dev_id) +{ + struct xdma_user_irq *user_irq; + + dbg_irq("(irq=%d) <<<< INTERRUPT SERVICE ROUTINE\n", irq); + + BUG_ON(!dev_id); + user_irq = (struct xdma_user_irq *)dev_id; + + return user_irq_service(irq, user_irq); +} + +/* + * xdma_channel_irq() - Interrupt handler for channel interrupts in MSI-X mode + * + * @dev_id pointer to xdma_dev + */ +static irqreturn_t xdma_channel_irq(int irq, void *dev_id) +{ + struct xdma_dev *xdev; + struct xdma_engine *engine; + struct interrupt_regs *irq_regs; + + dbg_irq("(irq=%d) <<<< INTERRUPT service ROUTINE\n", irq); + BUG_ON(!dev_id); + + engine = (struct xdma_engine *)dev_id; + xdev = engine->xdev; + + if (!xdev) { + WARN_ON(!xdev); + dbg_irq("xdma_channel_irq(irq=%d) xdev=%p ??\n", irq, xdev); + return IRQ_NONE; + } + + irq_regs = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + + /* Disable the interrupt for this engine */ + write_register(engine->interrupt_enable_mask_value, + &engine->regs->interrupt_enable_mask_w1c, + (unsigned long) + (&engine->regs->interrupt_enable_mask_w1c) - + (unsigned long)(&engine->regs)); + /* Dummy read to flush the above write */ + read_register(&irq_regs->channel_int_pending); + /* Schedule the bottom half */ + schedule_work(&engine->work); + + /* + * RTO - need to protect access here if multiple MSI-X are used for + * user interrupts + */ + xdev->irq_count++; + return IRQ_HANDLED; +} + +/* + * Unmap the BAR regions that had been mapped earlier using map_bars() + */ +static void unmap_bars(struct xdma_dev *xdev, struct pci_dev *dev) +{ + int i; + + for (i = 0; i < XDMA_BAR_NUM; i++) { + /* is this BAR mapped? */ + if (xdev->bar[i]) { + /* unmap BAR */ + pci_iounmap(dev, xdev->bar[i]); + /* mark as unmapped */ + xdev->bar[i] = NULL; + } + } +} + +static int map_single_bar(struct xdma_dev *xdev, struct pci_dev *dev, int idx) +{ + resource_size_t bar_start; + resource_size_t bar_len; + resource_size_t map_len; + + bar_start = pci_resource_start(dev, idx); + bar_len = pci_resource_len(dev, idx); + map_len = bar_len; + + xdev->bar[idx] = NULL; + + /* do not map BARs with length 0. Note that start MAY be 0! */ + if (!bar_len) { + //pr_info("BAR #%d is not present - skipping\n", idx); + return 0; + } + + /* BAR size exceeds maximum desired mapping? */ + if (bar_len > INT_MAX) { + pr_info("Limit BAR %d mapping from %llu to %d bytes\n", idx, + (u64)bar_len, INT_MAX); + map_len = (resource_size_t)INT_MAX; + } + /* + * map the full device memory or IO region into kernel virtual + * address space + */ + dbg_init("BAR%d: %llu bytes to be mapped.\n", idx, (u64)map_len); + xdev->bar[idx] = pci_iomap(dev, idx, map_len); + + if (!xdev->bar[idx]) { + pr_info("Could not map BAR %d.\n", idx); + return -1; + } + + pr_info("BAR%d at 0x%llx mapped at 0x%p, length=%llu(/%llu)\n", idx, + (u64)bar_start, xdev->bar[idx], (u64)map_len, (u64)bar_len); + + return (int)map_len; +} + +static int is_config_bar(struct xdma_dev *xdev, int idx) +{ + u32 irq_id = 0; + u32 cfg_id = 0; + int flag = 0; + u32 mask = 0xffff0000; /* Compare only XDMA ID's not Version number */ + struct interrupt_regs *irq_regs = + (struct interrupt_regs *) (xdev->bar[idx] + XDMA_OFS_INT_CTRL); + struct config_regs *cfg_regs = + (struct config_regs *)(xdev->bar[idx] + XDMA_OFS_CONFIG); + + irq_id = read_register(&irq_regs->identifier); + cfg_id = read_register(&cfg_regs->identifier); + + if (((irq_id & mask)== IRQ_BLOCK_ID) && + ((cfg_id & mask)== CONFIG_BLOCK_ID)) { + dbg_init("BAR %d is the XDMA config BAR\n", idx); + flag = 1; + } else { + dbg_init("BAR %d is NOT the XDMA config BAR: 0x%x, 0x%x.\n", + idx, irq_id, cfg_id); + flag = 0; + } + + return flag; +} + +static void identify_bars(struct xdma_dev *xdev, int *bar_id_list, int num_bars, + int config_bar_pos) +{ + /* + * The following logic identifies which BARs contain what functionality + * based on the position of the XDMA config BAR and the number of BARs + * detected. The rules are that the user logic and bypass logic BARs + * are optional. When both are present, the XDMA config BAR will be the + * 2nd BAR detected (config_bar_pos = 1), with the user logic being + * detected first and the bypass being detected last. When one is + * omitted, the type of BAR present can be identified by whether the + * XDMA config BAR is detected first or last. When both are omitted, + * only the XDMA config BAR is present. This somewhat convoluted + * approach is used instead of relying on BAR numbers in order to work + * correctly with both 32-bit and 64-bit BARs. + */ + + BUG_ON(!xdev); + BUG_ON(!bar_id_list); + + dbg_init("xdev 0x%p, bars %d, config at %d.\n", + xdev, num_bars, config_bar_pos); + + switch (num_bars) { + case 1: + /* Only one BAR present - no extra work necessary */ + break; + + case 2: + if (config_bar_pos == 0) { + xdev->bypass_bar_idx = bar_id_list[1]; + } else if (config_bar_pos == 1) { + xdev->user_bar_idx = bar_id_list[0]; + } else { + pr_info("2, XDMA config BAR unexpected %d.\n", + config_bar_pos); + } + break; + + case 3: + case 4: + if ((config_bar_pos == 1) || (config_bar_pos == 2)) { + /* user bar at bar #0 */ + xdev->user_bar_idx = bar_id_list[0]; + /* bypass bar at the last bar */ + xdev->bypass_bar_idx = bar_id_list[num_bars - 1]; + } else { + pr_info("3/4, XDMA config BAR unexpected %d.\n", + config_bar_pos); + } + break; + + default: + /* Should not occur - warn user but safe to continue */ + pr_info("Unexpected # BARs (%d), XDMA config BAR only.\n", + num_bars); + break; + + } + pr_info("%d BARs: config %d, user %d, bypass %d.\n", + num_bars, config_bar_pos, xdev->user_bar_idx, + xdev->bypass_bar_idx); +} + +/* map_bars() -- map device regions into kernel virtual address space + * + * Map the device memory regions into kernel virtual address space after + * verifying their sizes respect the minimum sizes needed + */ +static int map_bars(struct xdma_dev *xdev, struct pci_dev *dev) +{ + int rv; + int i; + int bar_id_list[XDMA_BAR_NUM]; + int bar_id_idx = 0; + int config_bar_pos = 0; + + /* iterate through all the BARs */ + for (i = 0; i < XDMA_BAR_NUM; i++) { + int bar_len; + + bar_len = map_single_bar(xdev, dev, i); + if (bar_len == 0) { + continue; + } else if (bar_len < 0) { + rv = -EINVAL; + goto fail; + } + + /* Try to identify BAR as XDMA control BAR */ + if ((bar_len >= XDMA_BAR_SIZE) && (xdev->config_bar_idx < 0)) { + + if (is_config_bar(xdev, i)) { + xdev->config_bar_idx = i; + config_bar_pos = bar_id_idx; + pr_info("config bar %d, pos %d.\n", + xdev->config_bar_idx, config_bar_pos); + } + } + + bar_id_list[bar_id_idx] = i; + bar_id_idx++; + } + + /* The XDMA config BAR must always be present */ + if (xdev->config_bar_idx < 0) { + pr_info("Failed to detect XDMA config BAR\n"); + rv = -EINVAL; + goto fail; + } + +#ifdef __LIBXDMA_CONFIG_BAR_ONLY__ + /* unmapped all other bars, except XDMA config. bar */ + for (i = 0; i < XDMA_BAR_NUM; i++) { + if (i == xdev->config_bar_idx) + continue; + + /* is this BAR mapped? */ + if (xdev->bar[i]) { + /* unmap BAR */ + pci_iounmap(dev, xdev->bar[i]); + /* mark as unmapped */ + xdev->bar[i] = NULL; + pr_info("unmap non-config bar %d.\n", i); + } + } +#else + identify_bars(xdev, bar_id_list, bar_id_idx, config_bar_pos); +#endif + + /* successfully mapped all required BAR regions */ + return 0; + +fail: + /* unwind; unmap any BARs that we did map */ + unmap_bars(xdev, dev); + return rv; +} + +/* + * MSI-X interrupt: + * <h2c+c2h channel_max> vectors, followed by <user_max> vectors + */ + +/* + * RTO - code to detect if MSI/MSI-X capability exists is derived + * from linux/pci/msi.c - pci_msi_check_device + */ + +#ifndef arch_msi_check_device +int arch_msi_check_device(struct pci_dev *dev, int nvec, int type) +{ + return 0; +} +#endif + +/* type = PCI_CAP_ID_MSI or PCI_CAP_ID_MSIX */ +static int msi_msix_capable(struct pci_dev *dev, int type) +{ + struct pci_bus *bus; + int ret; + + if (!dev || dev->no_msi) + return 0; + + for (bus = dev->bus; bus; bus = bus->parent) + if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) + return 0; + + ret = arch_msi_check_device(dev, 1, type); + if (ret) + return 0; + + if (!pci_find_capability(dev, type)) + return 0; + + return 1; +} + +static void disable_msi_msix(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + if (xdev->msix_enabled) { + pci_disable_msix(pdev); + xdev->msix_enabled = 0; + } else if (xdev->msi_enabled) { + pci_disable_msi(pdev); + xdev->msi_enabled = 0; + } +} + +static int enable_msi_msix(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + int rv = 0; + + BUG_ON(!xdev); + BUG_ON(!pdev); + + if (!interrupt_mode && msi_msix_capable(pdev, PCI_CAP_ID_MSIX)) { + int req_nvec = xdev->c2h_channel_max + xdev->h2c_channel_max + + xdev->user_max; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + dbg_init("Enabling MSI-X\n"); + rv = pci_alloc_irq_vectors(pdev, req_nvec, req_nvec, + PCI_IRQ_MSIX); +#else + int i; + + dbg_init("Enabling MSI-X\n"); + for (i = 0; i < req_nvec; i++) + xdev->entry[i].entry = i; + + rv = pci_enable_msix(pdev, xdev->entry, req_nvec); +#endif + if (rv < 0) + dbg_init("Couldn't enable MSI-X mode: %d\n", rv); + + xdev->msix_enabled = 1; + + } else if (interrupt_mode == 1 && + msi_msix_capable(pdev, PCI_CAP_ID_MSI)) { + /* enable message signalled interrupts */ + dbg_init("pci_enable_msi()\n"); + rv = pci_enable_msi(pdev); + if (rv < 0) + dbg_init("Couldn't enable MSI mode: %d\n", rv); + xdev->msi_enabled = 1; + + } else { + dbg_init("MSI/MSI-X not detected - using legacy interrupts\n"); + } + + return rv; +} + +static void pci_check_intr_pend(struct pci_dev *pdev) +{ + u16 v; + + pci_read_config_word(pdev, PCI_STATUS, &v); + if (v & PCI_STATUS_INTERRUPT) { + pr_info("%s PCI STATUS Interrupt pending 0x%x.\n", + dev_name(&pdev->dev), v); + pci_write_config_word(pdev, PCI_STATUS, PCI_STATUS_INTERRUPT); + } +} + +static void pci_keep_intx_enabled(struct pci_dev *pdev) +{ + /* workaround to a h/w bug: + * when msix/msi become unavaile, default to legacy. + * However the legacy enable was not checked. + * If the legacy was disabled, no ack then everything stuck + */ + u16 pcmd, pcmd_new; + + pci_read_config_word(pdev, PCI_COMMAND, &pcmd); + pcmd_new = pcmd & ~PCI_COMMAND_INTX_DISABLE; + if (pcmd_new != pcmd) { + pr_info("%s: clear INTX_DISABLE, 0x%x -> 0x%x.\n", + dev_name(&pdev->dev), pcmd, pcmd_new); + pci_write_config_word(pdev, PCI_COMMAND, pcmd_new); + } +} + +static void prog_irq_msix_user(struct xdma_dev *xdev, bool clear) +{ + /* user */ + struct interrupt_regs *int_regs = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + u32 i = xdev->c2h_channel_max + xdev->h2c_channel_max; + u32 max = i + xdev->user_max; + int j; + + for (j = 0; i < max; j++) { + u32 val = 0; + int k; + int shift = 0; + + if (clear) + i += 4; + else + for (k = 0; k < 4 && i < max; i++, k++, shift += 8) + val |= (i & 0x1f) << shift; + + write_register(val, &int_regs->user_msi_vector[j], + XDMA_OFS_INT_CTRL + + ((unsigned long)&int_regs->user_msi_vector[j] - + (unsigned long)int_regs)); + + dbg_init("vector %d, 0x%x.\n", j, val); + } +} + +static void prog_irq_msix_channel(struct xdma_dev *xdev, bool clear) +{ + struct interrupt_regs *int_regs = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + u32 max = xdev->c2h_channel_max + xdev->h2c_channel_max; + u32 i; + int j; + + /* engine */ + for (i = 0, j = 0; i < max; j++) { + u32 val = 0; + int k; + int shift = 0; + + if (clear) + i += 4; + else + for (k = 0; k < 4 && i < max; i++, k++, shift += 8) + val |= (i & 0x1f) << shift; + + write_register(val, &int_regs->channel_msi_vector[j], + XDMA_OFS_INT_CTRL + + ((unsigned long)&int_regs->channel_msi_vector[j] - + (unsigned long)int_regs)); + dbg_init("vector %d, 0x%x.\n", j, val); + } +} + +static void irq_msix_channel_teardown(struct xdma_dev *xdev) +{ + struct xdma_engine *engine; + int j = 0; + int i = 0; + + if (!xdev->msix_enabled) + return; + + prog_irq_msix_channel(xdev, 1); + + engine = xdev->engine_h2c; + for (i = 0; i < xdev->h2c_channel_max; i++, j++, engine++) { + if (!engine->msix_irq_line) + break; + dbg_sg("Release IRQ#%d for engine %p\n", engine->msix_irq_line, + engine); + free_irq(engine->msix_irq_line, engine); + } + + engine = xdev->engine_c2h; + for (i = 0; i < xdev->c2h_channel_max; i++, j++, engine++) { + if (!engine->msix_irq_line) + break; + dbg_sg("Release IRQ#%d for engine %p\n", engine->msix_irq_line, + engine); + free_irq(engine->msix_irq_line, engine); + } +} + +static int irq_msix_channel_setup(struct xdma_dev *xdev) +{ + int i; + int j = xdev->h2c_channel_max; + int rv = 0; + u32 vector; + struct xdma_engine *engine; + + BUG_ON(!xdev); + if (!xdev->msix_enabled) + return 0; + + engine = xdev->engine_h2c; + for (i = 0; i < xdev->h2c_channel_max; i++, engine++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + vector = pci_irq_vector(xdev->pdev, i); +#else + vector = xdev->entry[i].vector; +#endif + rv = request_irq(vector, xdma_channel_irq, 0, xdev->mod_name, + engine); + if (rv) { + pr_info("requesti irq#%d failed %d, engine %s.\n", + vector, rv, engine->name); + return rv; + } + pr_info("engine %s, irq#%d.\n", engine->name, vector); + engine->msix_irq_line = vector; + } + + engine = xdev->engine_c2h; + for (i = 0; i < xdev->c2h_channel_max; i++, j++, engine++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + vector = pci_irq_vector(xdev->pdev, j); +#else + vector = xdev->entry[j].vector; +#endif + rv = request_irq(vector, xdma_channel_irq, 0, xdev->mod_name, + engine); + if (rv) { + pr_info("requesti irq#%d failed %d, engine %s.\n", + vector, rv, engine->name); + return rv; + } + pr_info("engine %s, irq#%d.\n", engine->name, vector); + engine->msix_irq_line = vector; + } + + return 0; +} + +static void irq_msix_user_teardown(struct xdma_dev *xdev) +{ + int i; + int j = xdev->h2c_channel_max + xdev->c2h_channel_max; + + BUG_ON(!xdev); + + if (!xdev->msix_enabled) + return; + + prog_irq_msix_user(xdev, 1); + + for (i = 0; i < xdev->user_max; i++, j++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + u32 vector = pci_irq_vector(xdev->pdev, j); +#else + u32 vector = xdev->entry[j].vector; +#endif + dbg_init("user %d, releasing IRQ#%d\n", i, vector); + free_irq(vector, &xdev->user_irq[i]); + } +} + +static int irq_msix_user_setup(struct xdma_dev *xdev) +{ + int i; + int j = xdev->h2c_channel_max + xdev->c2h_channel_max; + int rv = 0; + + /* vectors set in probe_scan_for_msi() */ + for (i = 0; i < xdev->user_max; i++, j++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + u32 vector = pci_irq_vector(xdev->pdev, j); +#else + u32 vector = xdev->entry[j].vector; +#endif + rv = request_irq(vector, xdma_user_irq, 0, xdev->mod_name, + &xdev->user_irq[i]); + if (rv) { + pr_info("user %d couldn't use IRQ#%d, %d\n", + i, vector, rv); + break; + } + pr_info("%d-USR-%d, IRQ#%d with 0x%p\n", xdev->idx, i, vector, + &xdev->user_irq[i]); + } + + /* If any errors occur, free IRQs that were successfully requested */ + if (rv) { + for (i--, j--; i >= 0; i--, j--) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + u32 vector = pci_irq_vector(xdev->pdev, j); +#else + u32 vector = xdev->entry[j].vector; +#endif + free_irq(vector, &xdev->user_irq[i]); + } + } + + return rv; +} + +static int irq_msi_setup(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + int rv; + + xdev->irq_line = (int)pdev->irq; + rv = request_irq(pdev->irq, xdma_isr, 0, xdev->mod_name, xdev); + if (rv) + dbg_init("Couldn't use IRQ#%d, %d\n", pdev->irq, rv); + else + dbg_init("Using IRQ#%d with 0x%p\n", pdev->irq, xdev); + + return rv; +} + +static int irq_legacy_setup(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + u32 w; + u8 val; + void *reg; + int rv; + + pci_read_config_byte(pdev, PCI_INTERRUPT_PIN, &val); + dbg_init("Legacy Interrupt register value = %d\n", val); + if (val > 1) { + val--; + w = (val<<24) | (val<<16) | (val<<8)| val; + /* Program IRQ Block Channel vactor and IRQ Block User vector + * with Legacy interrupt value */ + reg = xdev->bar[xdev->config_bar_idx] + 0x2080; // IRQ user + write_register(w, reg, 0x2080); + write_register(w, reg+0x4, 0x2084); + write_register(w, reg+0x8, 0x2088); + write_register(w, reg+0xC, 0x208C); + reg = xdev->bar[xdev->config_bar_idx] + 0x20A0; // IRQ Block + write_register(w, reg, 0x20A0); + write_register(w, reg+0x4, 0x20A4); + } + + xdev->irq_line = (int)pdev->irq; + rv = request_irq(pdev->irq, xdma_isr, IRQF_SHARED, xdev->mod_name, + xdev); + if (rv) + dbg_init("Couldn't use IRQ#%d, %d\n", pdev->irq, rv); + else + dbg_init("Using IRQ#%d with 0x%p\n", pdev->irq, xdev); + + return rv; +} + +static void irq_teardown(struct xdma_dev *xdev) +{ + if (xdev->msix_enabled) { + irq_msix_channel_teardown(xdev); + irq_msix_user_teardown(xdev); + } else if (xdev->irq_line != -1) { + dbg_init("Releasing IRQ#%d\n", xdev->irq_line); + free_irq(xdev->irq_line, xdev); + } +} + +static int irq_setup(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + pci_keep_intx_enabled(pdev); + + if (xdev->msix_enabled) { + int rv = irq_msix_channel_setup(xdev); + if (rv) + return rv; + rv = irq_msix_user_setup(xdev); + if (rv) + return rv; + prog_irq_msix_channel(xdev, 0); + prog_irq_msix_user(xdev, 0); + + return 0; + } else if (xdev->msi_enabled) + return irq_msi_setup(xdev, pdev); + + return irq_legacy_setup(xdev, pdev); +} + +#ifdef __LIBXDMA_DEBUG__ +static void dump_desc(struct xdma_desc *desc_virt) +{ + int j; + u32 *p = (u32 *)desc_virt; + static char * const field_name[] = { + "magic|extra_adjacent|control", "bytes", "src_addr_lo", + "src_addr_hi", "dst_addr_lo", "dst_addr_hi", "next_addr", + "next_addr_pad"}; + char *dummy; + + /* remove warning about unused variable when debug printing is off */ + dummy = field_name[0]; + + for (j = 0; j < 8; j += 1) { + pr_info("0x%08lx/0x%02lx: 0x%08x 0x%08x %s\n", + (uintptr_t)p, (uintptr_t)p & 15, (int)*p, + le32_to_cpu(*p), field_name[j]); + p++; + } + pr_info("\n"); +} + +static void transfer_dump(struct xdma_transfer *transfer) +{ + int i; + struct xdma_desc *desc_virt = transfer->desc_virt; + + pr_info("xfer 0x%p, state 0x%x, f 0x%x, dir %d, len %u, last %d.\n", + transfer, transfer->state, transfer->flags, transfer->dir, + transfer->len, transfer->last_in_request); + + pr_info("transfer 0x%p, desc %d, bus 0x%llx, adj %d.\n", + transfer, transfer->desc_num, (u64)transfer->desc_bus, + transfer->desc_adjacent); + for (i = 0; i < transfer->desc_num; i += 1) + dump_desc(desc_virt + i); +} +#endif /* __LIBXDMA_DEBUG__ */ + +/* xdma_desc_alloc() - Allocate cache-coherent array of N descriptors. + * + * Allocates an array of 'number' descriptors in contiguous PCI bus addressable + * memory. Chains the descriptors as a singly-linked list; the descriptor's + * next * pointer specifies the bus address of the next descriptor. + * + * + * @dev Pointer to pci_dev + * @number Number of descriptors to be allocated + * @desc_bus_p Pointer where to store the first descriptor bus address + * + * @return Virtual address of the first descriptor + * + */ +static void transfer_desc_init(struct xdma_transfer *transfer, int count) +{ + struct xdma_desc *desc_virt = transfer->desc_virt; + dma_addr_t desc_bus = transfer->desc_bus; + int i; + int adj = count - 1; + int extra_adj; + u32 temp_control; + + BUG_ON(count > XDMA_TRANSFER_MAX_DESC); + + /* create singly-linked list for SG DMA controller */ + for (i = 0; i < count - 1; i++) { + /* increment bus address to next in array */ + desc_bus += sizeof(struct xdma_desc); + + /* singly-linked list uses bus addresses */ + desc_virt[i].next_lo = cpu_to_le32(PCI_DMA_L(desc_bus)); + desc_virt[i].next_hi = cpu_to_le32(PCI_DMA_H(desc_bus)); + desc_virt[i].bytes = cpu_to_le32(0); + + /* any adjacent descriptors? */ + if (adj > 0) { + extra_adj = adj - 1; + if (extra_adj > MAX_EXTRA_ADJ) + extra_adj = MAX_EXTRA_ADJ; + + adj--; + } else { + extra_adj = 0; + } + + temp_control = DESC_MAGIC | (extra_adj << 8); + + desc_virt[i].control = cpu_to_le32(temp_control); + } + /* { i = number - 1 } */ + /* zero the last descriptor next pointer */ + desc_virt[i].next_lo = cpu_to_le32(0); + desc_virt[i].next_hi = cpu_to_le32(0); + desc_virt[i].bytes = cpu_to_le32(0); + + temp_control = DESC_MAGIC; + + desc_virt[i].control = cpu_to_le32(temp_control); +} + +/* xdma_desc_link() - Link two descriptors + * + * Link the first descriptor to a second descriptor, or terminate the first. + * + * @first first descriptor + * @second second descriptor, or NULL if first descriptor must be set as last. + * @second_bus bus address of second descriptor + */ +static void xdma_desc_link(struct xdma_desc *first, struct xdma_desc *second, + dma_addr_t second_bus) +{ + /* + * remember reserved control in first descriptor, but zero + * extra_adjacent! + */ + /* RTO - what's this about? Shouldn't it be 0x0000c0ffUL? */ + u32 control = le32_to_cpu(first->control) & 0x0000f0ffUL; + /* second descriptor given? */ + if (second) { + /* + * link last descriptor of 1st array to first descriptor of + * 2nd array + */ + first->next_lo = cpu_to_le32(PCI_DMA_L(second_bus)); + first->next_hi = cpu_to_le32(PCI_DMA_H(second_bus)); + WARN_ON(first->next_hi); + /* no second descriptor given */ + } else { + /* first descriptor is the last */ + first->next_lo = 0; + first->next_hi = 0; + } + /* merge magic, extra_adjacent and control field */ + control |= DESC_MAGIC; + + /* write bytes and next_num */ + first->control = cpu_to_le32(control); +} + +/* xdma_desc_adjacent -- Set how many descriptors are adjacent to this one */ +static void xdma_desc_adjacent(struct xdma_desc *desc, int next_adjacent) +{ + int extra_adj = 0; + /* remember reserved and control bits */ + u32 control = le32_to_cpu(desc->control) & 0x0000f0ffUL; + u32 max_adj_4k = 0; + + if (next_adjacent > 0) { + extra_adj = next_adjacent - 1; + if (extra_adj > MAX_EXTRA_ADJ){ + extra_adj = MAX_EXTRA_ADJ; + } + max_adj_4k = (0x1000 - ((le32_to_cpu(desc->next_lo))&0xFFF))/32 - 1; + if (extra_adj>max_adj_4k) { + extra_adj = max_adj_4k; + } + if(extra_adj<0){ + printk("Warning: extra_adj<0, converting it to 0\n"); + extra_adj = 0; + } + } + /* merge adjacent and control field */ + control |= 0xAD4B0000UL | (extra_adj << 8); + /* write control and next_adjacent */ + desc->control = cpu_to_le32(control); +} + +/* xdma_desc_control -- Set complete control field of a descriptor. */ +static void xdma_desc_control_set(struct xdma_desc *first, u32 control_field) +{ + /* remember magic and adjacent number */ + u32 control = le32_to_cpu(first->control) & ~(LS_BYTE_MASK); + + BUG_ON(control_field & ~(LS_BYTE_MASK)); + /* merge adjacent and control field */ + control |= control_field; + /* write control and next_adjacent */ + first->control = cpu_to_le32(control); +} + +/* xdma_desc_clear -- Clear bits in control field of a descriptor. */ +static void xdma_desc_control_clear(struct xdma_desc *first, u32 clear_mask) +{ + /* remember magic and adjacent number */ + u32 control = le32_to_cpu(first->control); + + BUG_ON(clear_mask & ~(LS_BYTE_MASK)); + + /* merge adjacent and control field */ + control &= (~clear_mask); + /* write control and next_adjacent */ + first->control = cpu_to_le32(control); +} + +/* xdma_desc_done - recycle cache-coherent linked list of descriptors. + * + * @dev Pointer to pci_dev + * @number Number of descriptors to be allocated + * @desc_virt Pointer to (i.e. virtual address of) first descriptor in list + * @desc_bus Bus address of first descriptor in list + */ +static inline void xdma_desc_done(struct xdma_desc *desc_virt) +{ + memset(desc_virt, 0, XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc)); +} + +/* xdma_desc() - Fill a descriptor with the transfer details + * + * @desc pointer to descriptor to be filled + * @addr root complex address + * @ep_addr end point address + * @len number of bytes, must be a (non-negative) multiple of 4. + * @dir, dma direction + * is the end point address. If zero, vice versa. + * + * Does not modify the next pointer + */ +static void xdma_desc_set(struct xdma_desc *desc, dma_addr_t rc_bus_addr, + u64 ep_addr, int len, int dir) +{ + /* transfer length */ + desc->bytes = cpu_to_le32(len); + if (dir == DMA_TO_DEVICE) { + /* read from root complex memory (source address) */ + desc->src_addr_lo = cpu_to_le32(PCI_DMA_L(rc_bus_addr)); + desc->src_addr_hi = cpu_to_le32(PCI_DMA_H(rc_bus_addr)); + /* write to end point address (destination address) */ + desc->dst_addr_lo = cpu_to_le32(PCI_DMA_L(ep_addr)); + desc->dst_addr_hi = cpu_to_le32(PCI_DMA_H(ep_addr)); + } else { + /* read from end point address (source address) */ + desc->src_addr_lo = cpu_to_le32(PCI_DMA_L(ep_addr)); + desc->src_addr_hi = cpu_to_le32(PCI_DMA_H(ep_addr)); + /* write to root complex memory (destination address) */ + desc->dst_addr_lo = cpu_to_le32(PCI_DMA_L(rc_bus_addr)); + desc->dst_addr_hi = cpu_to_le32(PCI_DMA_H(rc_bus_addr)); + } +} + +/* + * should hold the engine->lock; + */ +static void transfer_abort(struct xdma_engine *engine, + struct xdma_transfer *transfer) +{ + struct xdma_transfer *head; + + BUG_ON(!engine); + BUG_ON(!transfer); + BUG_ON(transfer->desc_num == 0); + + pr_info("abort transfer 0x%p, desc %d, engine desc queued %d.\n", + transfer, transfer->desc_num, engine->desc_dequeued); + + head = list_entry(engine->transfer_list.next, struct xdma_transfer, + entry); + if (head == transfer) + list_del(engine->transfer_list.next); + else + pr_info("engine %s, transfer 0x%p NOT found, 0x%p.\n", + engine->name, transfer, head); + + if (transfer->state == TRANSFER_STATE_SUBMITTED) + transfer->state = TRANSFER_STATE_ABORTED; +} + +/* transfer_queue() - Queue a DMA transfer on the engine + * + * @engine DMA engine doing the transfer + * @transfer DMA transfer submitted to the engine + * + * Takes and releases the engine spinlock + */ +static int transfer_queue(struct xdma_engine *engine, + struct xdma_transfer *transfer) +{ + int rv = 0; + struct xdma_transfer *transfer_started; + struct xdma_dev *xdev; + unsigned long flags; + + BUG_ON(!engine); + BUG_ON(!engine->xdev); + BUG_ON(!transfer); + BUG_ON(transfer->desc_num == 0); + dbg_tfr("transfer_queue(transfer=0x%p).\n", transfer); + + xdev = engine->xdev; + if (xdma_device_flag_check(xdev, XDEV_FLAG_OFFLINE)) { + pr_info("dev 0x%p offline, transfer 0x%p not queued.\n", + xdev, transfer); + return -EBUSY; + } + + /* lock the engine state */ + spin_lock_irqsave(&engine->lock, flags); + + engine->prev_cpu = get_cpu(); + put_cpu(); + + /* engine is being shutdown; do not accept new transfers */ + if (engine->shutdown & ENGINE_SHUTDOWN_REQUEST) { + pr_info("engine %s offline, transfer 0x%p not queued.\n", + engine->name, transfer); + rv = -EBUSY; + goto shutdown; + } + + /* mark the transfer as submitted */ + transfer->state = TRANSFER_STATE_SUBMITTED; + /* add transfer to the tail of the engine transfer queue */ + list_add_tail(&transfer->entry, &engine->transfer_list); + + /* engine is idle? */ + if (!engine->running) { + /* start engine */ + dbg_tfr("transfer_queue(): starting %s engine.\n", + engine->name); + transfer_started = engine_start(engine); + dbg_tfr("transfer=0x%p started %s engine with transfer 0x%p.\n", + transfer, engine->name, transfer_started); + } else { + dbg_tfr("transfer=0x%p queued, with %s engine running.\n", + transfer, engine->name); + } + +shutdown: + /* unlock the engine state */ + dbg_tfr("engine->running = %d\n", engine->running); + spin_unlock_irqrestore(&engine->lock, flags); + return rv; +} + +static void engine_alignments(struct xdma_engine *engine) +{ + u32 w; + u32 align_bytes; + u32 granularity_bytes; + u32 address_bits; + + w = read_register(&engine->regs->alignments); + dbg_init("engine %p name %s alignments=0x%08x\n", engine, + engine->name, (int)w); + + /* RTO - add some macros to extract these fields */ + align_bytes = (w & 0x00ff0000U) >> 16; + granularity_bytes = (w & 0x0000ff00U) >> 8; + address_bits = (w & 0x000000ffU); + + dbg_init("align_bytes = %d\n", align_bytes); + dbg_init("granularity_bytes = %d\n", granularity_bytes); + dbg_init("address_bits = %d\n", address_bits); + + if (w) { + engine->addr_align = align_bytes; + engine->len_granularity = granularity_bytes; + engine->addr_bits = address_bits; + } else { + /* Some default values if alignments are unspecified */ + engine->addr_align = 1; + engine->len_granularity = 1; + engine->addr_bits = 64; + } +} + +static void engine_free_resource(struct xdma_engine *engine) +{ + struct xdma_dev *xdev = engine->xdev; + + /* Release memory use for descriptor writebacks */ + if (engine->poll_mode_addr_virt) { + dbg_sg("Releasing memory for descriptor writeback\n"); + dma_free_coherent(&xdev->pdev->dev, + sizeof(struct xdma_poll_wb), + engine->poll_mode_addr_virt, + engine->poll_mode_bus); + dbg_sg("Released memory for descriptor writeback\n"); + engine->poll_mode_addr_virt = NULL; + } + + if (engine->desc) { + dbg_init("device %s, engine %s pre-alloc desc 0x%p,0x%llx.\n", + dev_name(&xdev->pdev->dev), engine->name, + engine->desc, engine->desc_bus); + dma_free_coherent(&xdev->pdev->dev, + XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc), + engine->desc, engine->desc_bus); + engine->desc = NULL; + } + + if (engine->cyclic_result) { + dma_free_coherent(&xdev->pdev->dev, + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result), + engine->cyclic_result, engine->cyclic_result_bus); + engine->cyclic_result = NULL; + } +} + +static void engine_destroy(struct xdma_dev *xdev, struct xdma_engine *engine) +{ + BUG_ON(!xdev); + BUG_ON(!engine); + + dbg_sg("Shutting down engine %s%d", engine->name, engine->channel); + + /* Disable interrupts to stop processing new events during shutdown */ + write_register(0x0, &engine->regs->interrupt_enable_mask, + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); + + if (enable_credit_mp && engine->streaming && + engine->dir == DMA_FROM_DEVICE) { + u32 reg_value = (0x1 << engine->channel) << 16; + struct sgdma_common_regs *reg = (struct sgdma_common_regs *) + (xdev->bar[xdev->config_bar_idx] + + (0x6*TARGET_SPACING)); + write_register(reg_value, ®->credit_mode_enable_w1c, 0); + } + + /* Release memory use for descriptor writebacks */ + engine_free_resource(engine); + + memset(engine, 0, sizeof(struct xdma_engine)); + /* Decrement the number of engines available */ + xdev->engines_num--; +} + +/** + *engine_cyclic_stop() - stop a cyclic transfer running on an SG DMA engine + * + *engine->lock must be taken + */ +struct xdma_transfer *engine_cyclic_stop(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer = 0; + + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) { + /* pick first transfer on the queue (was submitted to engine) */ + transfer = list_entry(engine->transfer_list.next, + struct xdma_transfer, entry); + BUG_ON(!transfer); + + xdma_engine_stop(engine); + + if (transfer->cyclic) { + if (engine->xdma_perf) + dbg_perf("Stopping perf transfer on %s\n", + engine->name); + else + dbg_perf("Stopping cyclic transfer on %s\n", + engine->name); + /* make sure the handler sees correct transfer state */ + transfer->cyclic = 1; + /* + * set STOP flag and interrupt on completion, on the + * last descriptor + */ + xdma_desc_control_set( + transfer->desc_virt + transfer->desc_num - 1, + XDMA_DESC_COMPLETED | XDMA_DESC_STOPPED); + } else { + dbg_sg("(engine=%p) running transfer is not cyclic\n", + engine); + } + } else { + dbg_sg("(engine=%p) found not running transfer.\n", engine); + } + return transfer; +} +EXPORT_SYMBOL_GPL(engine_cyclic_stop); + +static int engine_writeback_setup(struct xdma_engine *engine) +{ + u32 w; + struct xdma_dev *xdev; + struct xdma_poll_wb *writeback; + + BUG_ON(!engine); + xdev = engine->xdev; + BUG_ON(!xdev); + + /* + * RTO - doing the allocation per engine is wasteful since a full page + * is allocated each time - better to allocate one page for the whole + * device during probe() and set per-engine offsets here + */ + writeback = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + writeback->completed_desc_count = 0; + + dbg_init("Setting writeback location to 0x%llx for engine %p", + engine->poll_mode_bus, engine); + w = cpu_to_le32(PCI_DMA_L(engine->poll_mode_bus)); + write_register(w, &engine->regs->poll_mode_wb_lo, + (unsigned long)(&engine->regs->poll_mode_wb_lo) - + (unsigned long)(&engine->regs)); + w = cpu_to_le32(PCI_DMA_H(engine->poll_mode_bus)); + write_register(w, &engine->regs->poll_mode_wb_hi, + (unsigned long)(&engine->regs->poll_mode_wb_hi) - + (unsigned long)(&engine->regs)); + + return 0; +} + + +/* engine_create() - Create an SG DMA engine bookkeeping data structure + * + * An SG DMA engine consists of the resources for a single-direction transfer + * queue; the SG DMA hardware, the software queue and interrupt handling. + * + * @dev Pointer to pci_dev + * @offset byte address offset in BAR[xdev->config_bar_idx] resource for the + * SG DMA * controller registers. + * @dir: DMA_TO/FROM_DEVICE + * @streaming Whether the engine is attached to AXI ST (rather than MM) + */ +static int engine_init_regs(struct xdma_engine *engine) +{ + u32 reg_value; + int rv = 0; + + write_register(XDMA_CTRL_NON_INCR_ADDR, &engine->regs->control_w1c, + (unsigned long)(&engine->regs->control_w1c) - + (unsigned long)(&engine->regs)); + + engine_alignments(engine); + + /* Configure error interrupts by default */ + reg_value = XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + reg_value |= XDMA_CTRL_IE_MAGIC_STOPPED; + reg_value |= XDMA_CTRL_IE_MAGIC_STOPPED; + reg_value |= XDMA_CTRL_IE_READ_ERROR; + reg_value |= XDMA_CTRL_IE_DESC_ERROR; + + /* if using polled mode, configure writeback address */ + if (poll_mode) { + rv = engine_writeback_setup(engine); + if (rv) { + dbg_init("%s descr writeback setup failed.\n", + engine->name); + goto fail_wb; + } + } else { + /* enable the relevant completion interrupts */ + reg_value |= XDMA_CTRL_IE_DESC_STOPPED; + reg_value |= XDMA_CTRL_IE_DESC_COMPLETED; + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) + reg_value |= XDMA_CTRL_IE_IDLE_STOPPED; + } + + /* Apply engine configurations */ + write_register(reg_value, &engine->regs->interrupt_enable_mask, + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); + + engine->interrupt_enable_mask_value = reg_value; + + /* only enable credit mode for AXI-ST C2H */ + if (enable_credit_mp && engine->streaming && + engine->dir == DMA_FROM_DEVICE) { + + struct xdma_dev *xdev = engine->xdev; + u32 reg_value = (0x1 << engine->channel) << 16; + struct sgdma_common_regs *reg = (struct sgdma_common_regs *) + (xdev->bar[xdev->config_bar_idx] + + (0x6*TARGET_SPACING)); + + write_register(reg_value, ®->credit_mode_enable_w1s, 0); + } + + return 0; + +fail_wb: + return rv; +} + +static int engine_alloc_resource(struct xdma_engine *engine) +{ + struct xdma_dev *xdev = engine->xdev; + + engine->desc = dma_alloc_coherent(&xdev->pdev->dev, + XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc), + &engine->desc_bus, GFP_KERNEL); + if (!engine->desc) { + pr_warn("dev %s, %s pre-alloc desc OOM.\n", + dev_name(&xdev->pdev->dev), engine->name); + goto err_out; + } + + if (poll_mode) { + engine->poll_mode_addr_virt = dma_alloc_coherent( + &xdev->pdev->dev, + sizeof(struct xdma_poll_wb), + &engine->poll_mode_bus, GFP_KERNEL); + if (!engine->poll_mode_addr_virt) { + pr_warn("%s, %s poll pre-alloc writeback OOM.\n", + dev_name(&xdev->pdev->dev), engine->name); + goto err_out; + } + } + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { + engine->cyclic_result = dma_alloc_coherent(&xdev->pdev->dev, + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result), + &engine->cyclic_result_bus, GFP_KERNEL); + + if (!engine->cyclic_result) { + pr_warn("%s, %s pre-alloc result OOM.\n", + dev_name(&xdev->pdev->dev), engine->name); + goto err_out; + } + } + + return 0; + +err_out: + engine_free_resource(engine); + return -ENOMEM; +} + +static int engine_init(struct xdma_engine *engine, struct xdma_dev *xdev, + int offset, enum dma_data_direction dir, int channel) +{ + int rv; + u32 val; + + dbg_init("channel %d, offset 0x%x, dir %d.\n", channel, offset, dir); + + /* set magic */ + engine->magic = MAGIC_ENGINE; + + engine->channel = channel; + + /* engine interrupt request bit */ + engine->irq_bitmask = (1 << XDMA_ENG_IRQ_NUM) - 1; + engine->irq_bitmask <<= (xdev->engines_num * XDMA_ENG_IRQ_NUM); + engine->bypass_offset = xdev->engines_num * BYPASS_MODE_SPACING; + + /* parent */ + engine->xdev = xdev; + /* register address */ + engine->regs = (xdev->bar[xdev->config_bar_idx] + offset); + engine->sgdma_regs = xdev->bar[xdev->config_bar_idx] + offset + + SGDMA_OFFSET_FROM_CHANNEL; + val = read_register(&engine->regs->identifier); + if (val & 0x8000U) + engine->streaming = 1; + + /* remember SG DMA direction */ + engine->dir = dir; + sprintf(engine->name, "%d-%s%d-%s", xdev->idx, + (dir == DMA_TO_DEVICE) ? "H2C" : "C2H", channel, + engine->streaming ? "ST" : "MM"); + + dbg_init("engine %p name %s irq_bitmask=0x%08x\n", engine, engine->name, + (int)engine->irq_bitmask); + + /* initialize the deferred work for transfer completion */ + INIT_WORK(&engine->work, engine_service_work); + + if (dir == DMA_TO_DEVICE) + xdev->mask_irq_h2c |= engine->irq_bitmask; + else + xdev->mask_irq_c2h |= engine->irq_bitmask; + xdev->engines_num++; + + rv = engine_alloc_resource(engine); + if (rv) + return rv; + + rv = engine_init_regs(engine); + if (rv) + return rv; + + return 0; +} + +/* transfer_destroy() - free transfer */ +static void transfer_destroy(struct xdma_dev *xdev, struct xdma_transfer *xfer) +{ + /* free descriptors */ + xdma_desc_done(xfer->desc_virt); + + if (xfer->last_in_request && (xfer->flags & XFER_FLAG_NEED_UNMAP)) { + struct sg_table *sgt = xfer->sgt; + + if (sgt->nents) { + pci_unmap_sg(xdev->pdev, sgt->sgl, sgt->nents, + xfer->dir); + sgt->nents = 0; + } + } +} + +static int transfer_build(struct xdma_engine *engine, + struct xdma_request_cb *req, unsigned int desc_max) +{ + struct xdma_transfer *xfer = &req->xfer; + struct sw_desc *sdesc = &(req->sdesc[req->sw_desc_idx]); + int i = 0; + int j = 0; + + for (; i < desc_max; i++, j++, sdesc++) { + dbg_desc("sw desc %d/%u: 0x%llx, 0x%x, ep 0x%llx.\n", + i + req->sw_desc_idx, req->sw_desc_cnt, + sdesc->addr, sdesc->len, req->ep_addr); + + /* fill in descriptor entry j with transfer details */ + xdma_desc_set(xfer->desc_virt + j, sdesc->addr, req->ep_addr, + sdesc->len, xfer->dir); + xfer->len += sdesc->len; + + /* for non-inc-add mode don't increment ep_addr */ + if (!engine->non_incr_addr) + req->ep_addr += sdesc->len; + } + req->sw_desc_idx += desc_max; + return 0; +} + +static int transfer_init(struct xdma_engine *engine, struct xdma_request_cb *req) +{ + struct xdma_transfer *xfer = &req->xfer; + unsigned int desc_max = min_t(unsigned int, + req->sw_desc_cnt - req->sw_desc_idx, + XDMA_TRANSFER_MAX_DESC); + int i = 0; + int last = 0; + u32 control; + + memset(xfer, 0, sizeof(*xfer)); + + /* initialize wait queue */ + init_waitqueue_head(&xfer->wq); + + /* remember direction of transfer */ + xfer->dir = engine->dir; + + xfer->desc_virt = engine->desc; + xfer->desc_bus = engine->desc_bus; + + transfer_desc_init(xfer, desc_max); + + dbg_sg("transfer->desc_bus = 0x%llx.\n", (u64)xfer->desc_bus); + + transfer_build(engine, req, desc_max); + + /* terminate last descriptor */ + last = desc_max - 1; + xdma_desc_link(xfer->desc_virt + last, 0, 0); + /* stop engine, EOP for AXI ST, req IRQ on last descriptor */ + control = XDMA_DESC_STOPPED; + control |= XDMA_DESC_EOP; + control |= XDMA_DESC_COMPLETED; + xdma_desc_control_set(xfer->desc_virt + last, control); + + xfer->desc_num = xfer->desc_adjacent = desc_max; + + dbg_sg("transfer 0x%p has %d descriptors\n", xfer, xfer->desc_num); + /* fill in adjacent numbers */ + for (i = 0; i < xfer->desc_num; i++) + xdma_desc_adjacent(xfer->desc_virt + i, xfer->desc_num - i - 1); + + return 0; +} + +#ifdef __LIBXDMA_DEBUG__ +static void sgt_dump(struct sg_table *sgt) +{ + int i; + struct scatterlist *sg = sgt->sgl; + + pr_info("sgt 0x%p, sgl 0x%p, nents %u/%u.\n", + sgt, sgt->sgl, sgt->nents, sgt->orig_nents); + + for (i = 0; i < sgt->orig_nents; i++, sg = sg_next(sg)) + pr_info("%d, 0x%p, pg 0x%p,%u+%u, dma 0x%llx,%u.\n", + i, sg, sg_page(sg), sg->offset, sg->length, + sg_dma_address(sg), sg_dma_len(sg)); +} + +static void xdma_request_cb_dump(struct xdma_request_cb *req) +{ + int i; + + pr_info("request 0x%p, total %u, ep 0x%llx, sw_desc %u, sgt 0x%p.\n", + req, req->total_len, req->ep_addr, req->sw_desc_cnt, req->sgt); + sgt_dump(req->sgt); + for (i = 0; i < req->sw_desc_cnt; i++) + pr_info("%d/%u, 0x%llx, %u.\n", + i, req->sw_desc_cnt, req->sdesc[i].addr, + req->sdesc[i].len); +} +#endif + +static void xdma_request_free(struct xdma_request_cb *req) +{ + if (((unsigned long)req) >= VMALLOC_START && + ((unsigned long)req) < VMALLOC_END) + vfree(req); + else + kfree(req); +} + +static struct xdma_request_cb * xdma_request_alloc(unsigned int sdesc_nr) +{ + struct xdma_request_cb *req; + unsigned int size = sizeof(struct xdma_request_cb) + + sdesc_nr * sizeof(struct sw_desc); + + req = kzalloc(size, GFP_KERNEL); + if (!req) { + req = vmalloc(size); + if (req) + memset(req, 0, size); + } + if (!req) { + pr_info("OOM, %u sw_desc, %u.\n", sdesc_nr, size); + return NULL; + } + + return req; +} + +static struct xdma_request_cb * xdma_init_request(struct sg_table *sgt, + u64 ep_addr) +{ + struct xdma_request_cb *req; + struct scatterlist *sg = sgt->sgl; + int max = sgt->nents; + int extra = 0; + int i, j = 0; + + for (i = 0; i < max; i++, sg = sg_next(sg)) { + unsigned int len = sg_dma_len(sg); + + if (unlikely(len > desc_blen_max)) + extra += (len + desc_blen_max - 1) / desc_blen_max; + } + +//pr_info("ep 0x%llx, desc %u+%u.\n", ep_addr, max, extra); + + max += extra; + req = xdma_request_alloc(max); + if (!req) + return NULL; + + req->sgt = sgt; + req->ep_addr = ep_addr; + + for (i = 0, sg = sgt->sgl; i < sgt->nents; i++, sg = sg_next(sg)) { + unsigned int tlen = sg_dma_len(sg); + dma_addr_t addr = sg_dma_address(sg); + + req->total_len += tlen; + while (tlen) { + req->sdesc[j].addr = addr; + if (tlen > desc_blen_max) { + req->sdesc[j].len = desc_blen_max; + addr += desc_blen_max; + tlen -= desc_blen_max; + } else { + req->sdesc[j].len = tlen; + tlen = 0; + } + j++; + } + } + BUG_ON(j > max); + + req->sw_desc_cnt = j; +#ifdef __LIBXDMA_DEBUG__ + xdma_request_cb_dump(req); +#endif + return req; +} + +ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, + struct sg_table *sgt, bool dma_mapped, int timeout_ms) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + struct xdma_engine *engine; + int rv = 0; + ssize_t done = 0; + struct scatterlist *sg = sgt->sgl; + int nents; + enum dma_data_direction dir = write ? DMA_TO_DEVICE : DMA_FROM_DEVICE; + struct xdma_request_cb *req = NULL; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + if (write == 1) { + if (channel >= xdev->h2c_channel_max) { + pr_warn("H2C channel %d >= %d.\n", + channel, xdev->h2c_channel_max); + return -EINVAL; + } + engine = &xdev->engine_h2c[channel]; + } else if (write == 0) { + if (channel >= xdev->c2h_channel_max) { + pr_warn("C2H channel %d >= %d.\n", + channel, xdev->c2h_channel_max); + return -EINVAL; + } + engine = &xdev->engine_c2h[channel]; + } else { + pr_warn("write %d, exp. 0|1.\n", write); + return -EINVAL; + } + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + xdev = engine->xdev; + if (xdma_device_flag_check(xdev, XDEV_FLAG_OFFLINE)) { + pr_info("xdev 0x%p, offline.\n", xdev); + return -EBUSY; + } + + /* check the direction */ + if (engine->dir != dir) { + pr_info("0x%p, %s, %d, W %d, 0x%x/0x%x mismatch.\n", + engine, engine->name, channel, write, engine->dir, dir); + return -EINVAL; + } + + if (!dma_mapped) { + nents = pci_map_sg(xdev->pdev, sg, sgt->orig_nents, dir); + if (!nents) { + pr_info("map sgl failed, sgt 0x%p.\n", sgt); + return -EIO; + } + sgt->nents = nents; + } else { + BUG_ON(!sgt->nents); + } + + req = xdma_init_request(sgt, ep_addr); + if (!req) { + rv = -ENOMEM; + goto unmap_sgl; + } + + dbg_tfr("%s, len %u sg cnt %u.\n", + engine->name, req->total_len, req->sw_desc_cnt); + + sg = sgt->sgl; + nents = req->sw_desc_cnt; + while (nents) { + unsigned long flags; + struct xdma_transfer *xfer; + + /* one transfer at a time */ + spin_lock(&engine->desc_lock); + + /* build transfer */ + rv = transfer_init(engine, req); + if (rv < 0) { + spin_unlock(&engine->desc_lock); + goto unmap_sgl; + } + xfer = &req->xfer; + + if (!dma_mapped) + xfer->flags = XFER_FLAG_NEED_UNMAP; + + /* last transfer for the given request? */ + nents -= xfer->desc_num; + if (!nents) { + xfer->last_in_request = 1; + xfer->sgt = sgt; + } + + dbg_tfr("xfer, %u, ep 0x%llx, done %lu, sg %u/%u.\n", + xfer->len, req->ep_addr, done, req->sw_desc_idx, + req->sw_desc_cnt); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); +#endif + + rv = transfer_queue(engine, xfer); + if (rv < 0) { + spin_unlock(&engine->desc_lock); + pr_info("unable to submit %s, %d.\n", engine->name, rv); + goto unmap_sgl; + } + + /* + * When polling, determine how many descriptors have been queued * on the engine to determine the writeback value expected + */ + if (poll_mode) { + unsigned int desc_count; + + spin_lock_irqsave(&engine->lock, flags); + desc_count = xfer->desc_num; + spin_unlock_irqrestore(&engine->lock, flags); + + dbg_tfr("%s poll desc_count=%d\n", + engine->name, desc_count); + rv = engine_service_poll(engine, desc_count); + + } else { + rv = wait_event_interruptible_timeout(xfer->wq, + (xfer->state != TRANSFER_STATE_SUBMITTED), + msecs_to_jiffies(timeout_ms)); + } + + spin_lock_irqsave(&engine->lock, flags); + + switch(xfer->state) { + case TRANSFER_STATE_COMPLETED: + spin_unlock_irqrestore(&engine->lock, flags); + + dbg_tfr("transfer %p, %u, ep 0x%llx compl, +%lu.\n", + xfer, xfer->len, req->ep_addr - xfer->len, done); + done += xfer->len; + rv = 0; + break; + case TRANSFER_STATE_FAILED: + pr_info("xfer 0x%p,%u, failed, ep 0x%llx.\n", + xfer, xfer->len, req->ep_addr - xfer->len); + spin_unlock_irqrestore(&engine->lock, flags); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); + sgt_dump(sgt); +#endif + rv = -EIO; + break; + default: + /* transfer can still be in-flight */ + pr_info("xfer 0x%p,%u, s 0x%x timed out, ep 0x%llx.\n", + xfer, xfer->len, xfer->state, req->ep_addr); + engine_status_read(engine, 0, 1); + //engine_status_dump(engine); + transfer_abort(engine, xfer); + + xdma_engine_stop(engine); + spin_unlock_irqrestore(&engine->lock, flags); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); + sgt_dump(sgt); +#endif + rv = -ERESTARTSYS; + break; + } + + transfer_destroy(xdev, xfer); + spin_unlock(&engine->desc_lock); + + if (rv < 0) + goto unmap_sgl; + } /* while (sg) */ + +unmap_sgl: + if (!dma_mapped && sgt->nents) { + pci_unmap_sg(xdev->pdev, sgt->sgl, sgt->orig_nents, dir); + sgt->nents = 0; + } + + if (req) + xdma_request_free(req); + + if (rv < 0) + return rv; + + return done; +} +EXPORT_SYMBOL_GPL(xdma_xfer_submit); + +int xdma_performance_submit(struct xdma_dev *xdev, struct xdma_engine *engine) +{ + u8 *buffer_virt; + u32 max_consistent_size = 128 * 32 * 1024; /* 1024 pages, 4MB */ + dma_addr_t buffer_bus; /* bus address */ + struct xdma_transfer *transfer; + u64 ep_addr = 0; + int num_desc_in_a_loop = 128; + int size_in_desc = engine->xdma_perf->transfer_size; + int size = size_in_desc * num_desc_in_a_loop; + int i; + + BUG_ON(size_in_desc > max_consistent_size); + + if (size > max_consistent_size) { + size = max_consistent_size; + num_desc_in_a_loop = size / size_in_desc; + } + + buffer_virt = dma_alloc_coherent(&xdev->pdev->dev, size, + &buffer_bus, GFP_KERNEL); + + /* allocate transfer data structure */ + transfer = kzalloc(sizeof(struct xdma_transfer), GFP_KERNEL); + BUG_ON(!transfer); + + /* 0 = write engine (to_dev=0) , 1 = read engine (to_dev=1) */ + transfer->dir = engine->dir; + /* set number of descriptors */ + transfer->desc_num = num_desc_in_a_loop; + + /* allocate descriptor list */ + if (!engine->desc) { + engine->desc = dma_alloc_coherent(&xdev->pdev->dev, + num_desc_in_a_loop * sizeof(struct xdma_desc), + &engine->desc_bus, GFP_KERNEL); + BUG_ON(!engine->desc); + dbg_init("device %s, engine %s pre-alloc desc 0x%p,0x%llx.\n", + dev_name(&xdev->pdev->dev), engine->name, + engine->desc, engine->desc_bus); + } + transfer->desc_virt = engine->desc; + transfer->desc_bus = engine->desc_bus; + + transfer_desc_init(transfer, transfer->desc_num); + + dbg_sg("transfer->desc_bus = 0x%llx.\n", (u64)transfer->desc_bus); + + for (i = 0; i < transfer->desc_num; i++) { + struct xdma_desc *desc = transfer->desc_virt + i; + dma_addr_t rc_bus_addr = buffer_bus + size_in_desc * i; + + /* fill in descriptor entry with transfer details */ + xdma_desc_set(desc, rc_bus_addr, ep_addr, size_in_desc, + engine->dir); + } + + /* stop engine and request interrupt on last descriptor */ + xdma_desc_control_set(transfer->desc_virt, 0); + /* create a linked loop */ + xdma_desc_link(transfer->desc_virt + transfer->desc_num - 1, + transfer->desc_virt, transfer->desc_bus); + + transfer->cyclic = 1; + + /* initialize wait queue */ + init_waitqueue_head(&transfer->wq); + + //printk("=== Descriptor print for PERF \n"); + //transfer_dump(transfer); + + dbg_perf("Queueing XDMA I/O %s request for performance measurement.\n", + engine->dir ? "write (to dev)" : "read (from dev)"); + transfer_queue(engine, transfer); + return 0; + +} +EXPORT_SYMBOL_GPL(xdma_performance_submit); + +static struct xdma_dev *alloc_dev_instance(struct pci_dev *pdev) +{ + int i; + struct xdma_dev *xdev; + struct xdma_engine *engine; + + BUG_ON(!pdev); + + /* allocate zeroed device book keeping structure */ + xdev = kzalloc(sizeof(struct xdma_dev), GFP_KERNEL); + if (!xdev) { + pr_info("OOM, xdma_dev.\n"); + return NULL; + } + spin_lock_init(&xdev->lock); + + xdev->magic = MAGIC_DEVICE; + xdev->config_bar_idx = -1; + xdev->user_bar_idx = -1; + xdev->bypass_bar_idx = -1; + xdev->irq_line = -1; + + /* create a driver to device reference */ + xdev->pdev = pdev; + dbg_init("xdev = 0x%p\n", xdev); + + /* Set up data user IRQ data structures */ + for (i = 0; i < 16; i++) { + xdev->user_irq[i].xdev = xdev; + spin_lock_init(&xdev->user_irq[i].events_lock); + init_waitqueue_head(&xdev->user_irq[i].events_wq); + xdev->user_irq[i].handler = NULL; + xdev->user_irq[i].user_idx = i; /* 0 based */ + } + + engine = xdev->engine_h2c; + for (i = 0; i < XDMA_CHANNEL_NUM_MAX; i++, engine++) { + spin_lock_init(&engine->lock); + spin_lock_init(&engine->desc_lock); + INIT_LIST_HEAD(&engine->transfer_list); + init_waitqueue_head(&engine->shutdown_wq); + init_waitqueue_head(&engine->xdma_perf_wq); + } + + engine = xdev->engine_c2h; + for (i = 0; i < XDMA_CHANNEL_NUM_MAX; i++, engine++) { + spin_lock_init(&engine->lock); + spin_lock_init(&engine->desc_lock); + INIT_LIST_HEAD(&engine->transfer_list); + init_waitqueue_head(&engine->shutdown_wq); + init_waitqueue_head(&engine->xdma_perf_wq); + } + + return xdev; +} + +static int request_regions(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + int rv; + + BUG_ON(!xdev); + BUG_ON(!pdev); + + dbg_init("pci_request_regions()\n"); + rv = pci_request_regions(pdev, xdev->mod_name); + /* could not request all regions? */ + if (rv) { + dbg_init("pci_request_regions() = %d, device in use?\n", rv); + /* assume device is in use so do not disable it later */ + xdev->regions_in_use = 1; + } else { + xdev->got_regions = 1; + } + + return rv; +} + +static int set_dma_mask(struct pci_dev *pdev) +{ + BUG_ON(!pdev); + + dbg_init("sizeof(dma_addr_t) == %ld\n", sizeof(dma_addr_t)); + /* 64-bit addressing capability for XDMA? */ + if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + /* query for DMA transfer */ + /* @see Documentation/DMA-mapping.txt */ + dbg_init("pci_set_dma_mask()\n"); + /* use 64-bit DMA */ + dbg_init("Using a 64-bit DMA mask.\n"); + /* use 32-bit DMA for descriptors */ + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + /* use 64-bit DMA, 32-bit for consistent */ + } else if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) { + dbg_init("Could not set 64-bit DMA mask.\n"); + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + /* use 32-bit DMA */ + dbg_init("Using a 32-bit DMA mask.\n"); + } else { + dbg_init("No suitable DMA possible.\n"); + return -EINVAL; + } + + return 0; +} + +static u32 get_engine_channel_id(struct engine_regs *regs) +{ + u32 value; + + BUG_ON(!regs); + + value = read_register(®s->identifier); + + return (value & 0x00000f00U) >> 8; +} + +static u32 get_engine_id(struct engine_regs *regs) +{ + u32 value; + + BUG_ON(!regs); + + value = read_register(®s->identifier); + return (value & 0xffff0000U) >> 16; +} + +static void remove_engines(struct xdma_dev *xdev) +{ + struct xdma_engine *engine; + int i; + + BUG_ON(!xdev); + + /* iterate over channels */ + for (i = 0; i < xdev->h2c_channel_max; i++) { + engine = &xdev->engine_h2c[i]; + if (engine->magic == MAGIC_ENGINE) { + dbg_sg("Remove %s, %d", engine->name, i); + engine_destroy(xdev, engine); + dbg_sg("%s, %d removed", engine->name, i); + } + } + + for (i = 0; i < xdev->c2h_channel_max; i++) { + engine = &xdev->engine_c2h[i]; + if (engine->magic == MAGIC_ENGINE) { + dbg_sg("Remove %s, %d", engine->name, i); + engine_destroy(xdev, engine); + dbg_sg("%s, %d removed", engine->name, i); + } + } +} + +static int probe_for_engine(struct xdma_dev *xdev, enum dma_data_direction dir, + int channel) +{ + struct engine_regs *regs; + int offset = channel * CHANNEL_SPACING; + u32 engine_id; + u32 engine_id_expected; + u32 channel_id; + struct xdma_engine *engine; + int rv; + + /* register offset for the engine */ + /* read channels at 0x0000, write channels at 0x1000, + * channels at 0x100 interval */ + if (dir == DMA_TO_DEVICE) { + engine_id_expected = XDMA_ID_H2C; + engine = &xdev->engine_h2c[channel]; + } else { + offset += H2C_CHANNEL_OFFSET; + engine_id_expected = XDMA_ID_C2H; + engine = &xdev->engine_c2h[channel]; + } + + regs = xdev->bar[xdev->config_bar_idx] + offset; + engine_id = get_engine_id(regs); + channel_id = get_engine_channel_id(regs); + + if ((engine_id != engine_id_expected) || (channel_id != channel)) { + dbg_init("%s %d engine, reg off 0x%x, id mismatch 0x%x,0x%x," + "exp 0x%x,0x%x, SKIP.\n", + dir == DMA_TO_DEVICE ? "H2C" : "C2H", + channel, offset, engine_id, channel_id, + engine_id_expected, channel_id != channel); + return -EINVAL; + } + + dbg_init("found AXI %s %d engine, reg. off 0x%x, id 0x%x,0x%x.\n", + dir == DMA_TO_DEVICE ? "H2C" : "C2H", channel, + offset, engine_id, channel_id); + + /* allocate and initialize engine */ + rv = engine_init(engine, xdev, offset, dir, channel); + if (rv != 0) { + pr_info("failed to create AXI %s %d engine.\n", + dir == DMA_TO_DEVICE ? "H2C" : "C2H", + channel); + return rv; + } + + return 0; +} + +static int probe_engines(struct xdma_dev *xdev) +{ + int i; + int rv = 0; + + BUG_ON(!xdev); + + /* iterate over channels */ + for (i = 0; i < xdev->h2c_channel_max; i++) { + rv = probe_for_engine(xdev, DMA_TO_DEVICE, i); + if (rv) + break; + } + xdev->h2c_channel_max = i; + + for (i = 0; i < xdev->c2h_channel_max; i++) { + rv = probe_for_engine(xdev, DMA_FROM_DEVICE, i); + if (rv) + break; + } + xdev->c2h_channel_max = i; + + return 0; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0) +static void pci_enable_relaxed_ordering(struct pci_dev *pdev) +{ + pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN); +} +#else +static void pci_enable_relaxed_ordering(struct pci_dev *pdev) +{ + u16 v; + int pos; + + pos = pci_pcie_cap(pdev); + if (pos > 0) { + pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &v); + v |= PCI_EXP_DEVCTL_RELAX_EN; + pci_write_config_word(pdev, pos + PCI_EXP_DEVCTL, v); + } +} +#endif + +static void pci_check_extended_tag(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + u16 cap; + u32 v; + void *__iomem reg; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0) + pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &cap); +#else + int pos; + + pos = pci_pcie_cap(pdev); + if (pos > 0) + pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &cap); + else { + pr_info("pdev 0x%p, unable to access pcie cap.\n", pdev); + return; + } +#endif + + if ((cap & PCI_EXP_DEVCTL_EXT_TAG)) + return; + + /* extended tag not enabled */ + pr_info("0x%p EXT_TAG disabled.\n", pdev); + + if (xdev->config_bar_idx < 0) { + pr_info("pdev 0x%p, xdev 0x%p, config bar UNKNOWN.\n", + pdev, xdev); + return; + } + + reg = xdev->bar[xdev->config_bar_idx] + XDMA_OFS_CONFIG + 0x4C; + v = read_register(reg); + v = (v & 0xFF) | (((u32)32) << 8); + write_register(v, reg, XDMA_OFS_CONFIG + 0x4C); +} + +void *xdma_device_open(const char *mname, struct pci_dev *pdev, int *user_max, + int *h2c_channel_max, int *c2h_channel_max) +{ + struct xdma_dev *xdev = NULL; + int rv = 0; + + pr_info("%s device %s, 0x%p.\n", mname, dev_name(&pdev->dev), pdev); + + /* allocate zeroed device book keeping structure */ + xdev = alloc_dev_instance(pdev); + if (!xdev) + return NULL; + xdev->mod_name = mname; + xdev->user_max = *user_max; + xdev->h2c_channel_max = *h2c_channel_max; + xdev->c2h_channel_max = *c2h_channel_max; + + xdma_device_flag_set(xdev, XDEV_FLAG_OFFLINE); + xdev_list_add(xdev); + + if (xdev->user_max == 0 || xdev->user_max > MAX_USER_IRQ) + xdev->user_max = MAX_USER_IRQ; + if (xdev->h2c_channel_max == 0 || + xdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX) + xdev->h2c_channel_max = XDMA_CHANNEL_NUM_MAX; + if (xdev->c2h_channel_max == 0 || + xdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX) + xdev->c2h_channel_max = XDMA_CHANNEL_NUM_MAX; + + rv = pci_enable_device(pdev); + if (rv) { + dbg_init("pci_enable_device() failed, %d.\n", rv); + goto err_enable; + } + + /* keep INTx enabled */ + pci_check_intr_pend(pdev); + + /* enable relaxed ordering */ + pci_enable_relaxed_ordering(pdev); + + pci_check_extended_tag(xdev, pdev); + + /* force MRRS to be 512 */ + rv = pcie_set_readrq(pdev, 512); + if (rv) + pr_info("device %s, error set PCI_EXP_DEVCTL_READRQ: %d.\n", + dev_name(&pdev->dev), rv); + + /* enable bus master capability */ + pci_set_master(pdev); + + rv = request_regions(xdev, pdev); + if (rv) + goto err_regions; + + rv = map_bars(xdev, pdev); + if (rv) + goto err_map; + + rv = set_dma_mask(pdev); + if (rv) + goto err_mask; + + check_nonzero_interrupt_status(xdev); + /* explicitely zero all interrupt enable masks */ + channel_interrupts_disable(xdev, ~0); + user_interrupts_disable(xdev, ~0); + read_interrupts(xdev); + + rv = probe_engines(xdev); + if (rv) + goto err_engines; + + rv = enable_msi_msix(xdev, pdev); + if (rv < 0) + goto err_enable_msix; + + rv = irq_setup(xdev, pdev); + if (rv < 0) + goto err_interrupts; + + if (!poll_mode) + channel_interrupts_enable(xdev, ~0); + + /* Flush writes */ + read_interrupts(xdev); + + *user_max = xdev->user_max; + *h2c_channel_max = xdev->h2c_channel_max; + *c2h_channel_max = xdev->c2h_channel_max; + + xdma_device_flag_clear(xdev, XDEV_FLAG_OFFLINE); + return (void *)xdev; + +err_interrupts: + irq_teardown(xdev); +err_enable_msix: + disable_msi_msix(xdev, pdev); +err_engines: + remove_engines(xdev); +err_mask: + unmap_bars(xdev, pdev); +err_map: + if (xdev->got_regions) + pci_release_regions(pdev); +err_regions: + if (!xdev->regions_in_use) + pci_disable_device(pdev); +err_enable: + xdev_list_remove(xdev); + kfree(xdev); + return NULL; +} +EXPORT_SYMBOL_GPL(xdma_device_open); + +void xdma_device_close(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + dbg_init("pdev 0x%p, xdev 0x%p.\n", pdev, dev_hndl); + + if (!dev_hndl) + return; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return; + + dbg_sg("remove(dev = 0x%p) where pdev->dev.driver_data = 0x%p\n", + pdev, xdev); + if (xdev->pdev != pdev) { + dbg_sg("pci_dev(0x%lx) != pdev(0x%lx)\n", + (unsigned long)xdev->pdev, (unsigned long)pdev); + } + + channel_interrupts_disable(xdev, ~0); + user_interrupts_disable(xdev, ~0); + read_interrupts(xdev); + + irq_teardown(xdev); + disable_msi_msix(xdev, pdev); + + remove_engines(xdev); + unmap_bars(xdev, pdev); + + if (xdev->got_regions) { + dbg_init("pci_release_regions 0x%p.\n", pdev); + pci_release_regions(pdev); + } + + if (!xdev->regions_in_use) { + dbg_init("pci_disable_device 0x%p.\n", pdev); + pci_disable_device(pdev); + } + + xdev_list_remove(xdev); + + kfree(xdev); +} +EXPORT_SYMBOL_GPL(xdma_device_close); + +void xdma_device_offline(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + struct xdma_engine *engine; + int i; + + if (!dev_hndl) + return; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return; + +pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); + xdma_device_flag_set(xdev, XDEV_FLAG_OFFLINE); + + /* wait for all engines to be idle */ + for (i = 0; i < xdev->h2c_channel_max; i++) { + unsigned long flags; + + engine = &xdev->engine_h2c[i]; + + if (engine->magic == MAGIC_ENGINE) { + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown |= ENGINE_SHUTDOWN_REQUEST; + + xdma_engine_stop(engine); + engine->running = 0; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + for (i = 0; i < xdev->c2h_channel_max; i++) { + unsigned long flags; + + engine = &xdev->engine_c2h[i]; + if (engine->magic == MAGIC_ENGINE) { + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown |= ENGINE_SHUTDOWN_REQUEST; + + xdma_engine_stop(engine); + engine->running = 0; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + /* turn off interrupts */ + channel_interrupts_disable(xdev, ~0); + user_interrupts_disable(xdev, ~0); + read_interrupts(xdev); + irq_teardown(xdev); + + pr_info("xdev 0x%p, done.\n", xdev); +} +EXPORT_SYMBOL_GPL(xdma_device_offline); + +void xdma_device_online(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + struct xdma_engine *engine; + unsigned long flags; + int i; + + if (!dev_hndl) + return; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return; + +pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); + + for (i = 0; i < xdev->h2c_channel_max; i++) { + engine = &xdev->engine_h2c[i]; + if (engine->magic == MAGIC_ENGINE) { + engine_init_regs(engine); + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown &= ~ENGINE_SHUTDOWN_REQUEST; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + for (i = 0; i < xdev->c2h_channel_max; i++) { + engine = &xdev->engine_c2h[i]; + if (engine->magic == MAGIC_ENGINE) { + engine_init_regs(engine); + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown &= ~ENGINE_SHUTDOWN_REQUEST; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + /* re-write the interrupt table */ + if (!poll_mode) { + irq_setup(xdev, pdev); + + channel_interrupts_enable(xdev, ~0); + user_interrupts_enable(xdev, xdev->mask_irq_user); + read_interrupts(xdev); + } + + xdma_device_flag_clear(xdev, XDEV_FLAG_OFFLINE); +pr_info("xdev 0x%p, done.\n", xdev); +} +EXPORT_SYMBOL_GPL(xdma_device_online); + +int xdma_device_restart(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return -EINVAL; + + pr_info("NOT implemented, 0x%p.\n", xdev); + return -EINVAL; +} +EXPORT_SYMBOL_GPL(xdma_device_restart); + +int xdma_user_isr_register(void *dev_hndl, unsigned int mask, + irq_handler_t handler, void *dev) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + int i; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + for (i = 0; i < xdev->user_max && mask; i++) { + unsigned int bit = (1 << i); + + if ((bit & mask) == 0) + continue; + + mask &= ~bit; + xdev->user_irq[i].handler = handler; + xdev->user_irq[i].dev = dev; + } + + return 0; +} +EXPORT_SYMBOL_GPL(xdma_user_isr_register); + +int xdma_user_isr_enable(void *dev_hndl, unsigned int mask) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + xdev->mask_irq_user |= mask; + /* enable user interrupts */ + user_interrupts_enable(xdev, mask); + read_interrupts(xdev); + + return 0; +} +EXPORT_SYMBOL_GPL(xdma_user_isr_enable); + +int xdma_user_isr_disable(void *dev_hndl, unsigned int mask) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + xdev->mask_irq_user &= ~mask; + user_interrupts_disable(xdev, mask); + read_interrupts(xdev); + + return 0; +} +EXPORT_SYMBOL_GPL(xdma_user_isr_disable); + +#ifdef __LIBXDMA_MOD__ +static int __init xdma_base_init(void) +{ + printk(KERN_INFO "%s", version); + return 0; +} + +static void __exit xdma_base_exit(void) +{ + return; +} + +module_init(xdma_base_init); +module_exit(xdma_base_exit); +#endif +/* makes an existing transfer cyclic */ +static void xdma_transfer_cyclic(struct xdma_transfer *transfer) +{ + /* link last descriptor to first descriptor */ + xdma_desc_link(transfer->desc_virt + transfer->desc_num - 1, + transfer->desc_virt, transfer->desc_bus); + /* remember transfer is cyclic */ + transfer->cyclic = 1; +} + +static int transfer_monitor_cyclic(struct xdma_engine *engine, + struct xdma_transfer *transfer, int timeout_ms) +{ + struct xdma_result *result; + int rc = 0; + + BUG_ON(!engine); + BUG_ON(!transfer); + + result = engine->cyclic_result; + BUG_ON(!result); + + if (poll_mode) { + int i ; + for (i = 0; i < 5; i++) { + rc = engine_service_poll(engine, 0); + if (rc) { + pr_info("%s service_poll failed %d.\n", + engine->name, rc); + rc = -ERESTARTSYS; + } + if (result[engine->rx_head].status) + return 0; + } + } else { + if (enable_credit_mp){ + dbg_tfr("%s: rx_head=%d,rx_tail=%d, wait ...\n", + engine->name, engine->rx_head, engine->rx_tail); + rc = wait_event_interruptible_timeout( transfer->wq, + (engine->rx_head!=engine->rx_tail || + engine->rx_overrun), + msecs_to_jiffies(timeout_ms)); + dbg_tfr("%s: wait returns %d, rx %d/%d, overrun %d.\n", + engine->name, rc, engine->rx_head, + engine->rx_tail, engine->rx_overrun); + } else { + rc = wait_event_interruptible_timeout( transfer->wq, + engine->eop_found, + msecs_to_jiffies(timeout_ms)); + dbg_tfr("%s: wait returns %d, eop_found %d.\n", + engine->name, rc, engine->eop_found); + } + } + + return 0; +} + +struct scatterlist *sglist_index(struct sg_table *sgt, unsigned int idx) +{ + struct scatterlist *sg = sgt->sgl; + int i; + + if (idx >= sgt->orig_nents) + return NULL; + + if (!idx) + return sg; + + for (i = 0; i < idx; i++, sg = sg_next(sg)) + ; + + return sg; +} + +static int copy_cyclic_to_user(struct xdma_engine *engine, int pkt_length, + int head, char __user *buf, size_t count) +{ + struct scatterlist *sg; + int more = pkt_length; + + BUG_ON(!engine); + BUG_ON(!buf); + + dbg_tfr("%s, pkt_len %d, head %d, user buf idx %u.\n", + engine->name, pkt_length, head, engine->user_buffer_index); + + sg = sglist_index(&engine->cyclic_sgt, head); + if (!sg) { + pr_info("%s, head %d OOR, sgl %u.\n", + engine->name, head, engine->cyclic_sgt.orig_nents); + return -EIO; + } + + /* EOP found? Transfer anything from head to EOP */ + while (more) { + unsigned int copy = more > PAGE_SIZE ? PAGE_SIZE : more; + unsigned int blen = count - engine->user_buffer_index; + int rv; + + if (copy > blen) + copy = blen; + + dbg_tfr("%s sg %d, 0x%p, copy %u to user %u.\n", + engine->name, head, sg, copy, + engine->user_buffer_index); + + rv = copy_to_user(&buf[engine->user_buffer_index], + page_address(sg_page(sg)), copy); + if (rv) { + pr_info("%s copy_to_user %u failed %d\n", + engine->name, copy, rv); + return -EIO; + } + + more -= copy; + engine->user_buffer_index += copy; + + if (engine->user_buffer_index == count) { + /* user buffer used up */ + break; + } + + head++; + if (head >= CYCLIC_RX_PAGES_MAX) { + head = 0; + sg = engine->cyclic_sgt.sgl; + } else + sg = sg_next(sg); + } + + return pkt_length; +} + +static int complete_cyclic(struct xdma_engine *engine, char __user *buf, + size_t count) +{ + struct xdma_result *result; + int pkt_length = 0; + int fault = 0; + int eop = 0; + int head; + int rc = 0; + int num_credit = 0; + unsigned long flags; + + BUG_ON(!engine); + result = engine->cyclic_result; + BUG_ON(!result); + + spin_lock_irqsave(&engine->lock, flags); + + /* where the host currently is in the ring buffer */ + head = engine->rx_head; + + /* iterate over newly received results */ + while (engine->rx_head != engine->rx_tail||engine->rx_overrun) { + + WARN_ON(result[engine->rx_head].status==0); + + dbg_tfr("%s, result[%d].status = 0x%x length = 0x%x.\n", + engine->name, engine->rx_head, + result[engine->rx_head].status, + result[engine->rx_head].length); + + if ((result[engine->rx_head].status >> 16) != C2H_WB) { + pr_info("%s, result[%d].status 0x%x, no magic.\n", + engine->name, engine->rx_head, + result[engine->rx_head].status); + fault = 1; + } else if (result[engine->rx_head].length > PAGE_SIZE) { + pr_info("%s, result[%d].len 0x%x, > PAGE_SIZE 0x%lx.\n", + engine->name, engine->rx_head, + result[engine->rx_head].length, PAGE_SIZE); + fault = 1; + } else if (result[engine->rx_head].length == 0) { + pr_info("%s, result[%d].length 0x%x.\n", + engine->name, engine->rx_head, + result[engine->rx_head].length); + fault = 1; + /* valid result */ + } else { + pkt_length += result[engine->rx_head].length; + num_credit++; + /* seen eop? */ + //if (result[engine->rx_head].status & RX_STATUS_EOP) + if (result[engine->rx_head].status & RX_STATUS_EOP){ + eop = 1; + engine->eop_found = 1; + } + + dbg_tfr("%s, pkt_length=%d (%s)\n", + engine->name, pkt_length, + eop ? "with EOP" : "no EOP yet"); + } + /* clear result */ + result[engine->rx_head].status = 0; + result[engine->rx_head].length = 0; + /* proceed head pointer so we make progress, even when fault */ + engine->rx_head = (engine->rx_head + 1) % CYCLIC_RX_PAGES_MAX; + + /* stop processing if a fault/eop was detected */ + if (fault || eop){ + break; + } + } + + spin_unlock_irqrestore(&engine->lock, flags); + + if (fault) + return -EIO; + + rc = copy_cyclic_to_user(engine, pkt_length, head, buf, count); + engine->rx_overrun = 0; + /* if copy is successful, release credits */ + if(rc > 0) + write_register(num_credit,&engine->sgdma_regs->credits, 0); + + return rc; +} + +ssize_t xdma_engine_read_cyclic(struct xdma_engine *engine, char __user *buf, + size_t count, int timeout_ms) +{ + int i = 0; + int rc = 0; + int rc_len = 0; + struct xdma_transfer *transfer; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + transfer = &engine->cyclic_req->xfer; + BUG_ON(!transfer); + + engine->user_buffer_index = 0; + + do { + rc = transfer_monitor_cyclic(engine, transfer, timeout_ms); + if (rc < 0) + return rc; + rc = complete_cyclic(engine, buf, count); + if (rc < 0) + return rc; + rc_len += rc; + + i++; + if (i > 10) + break; + } while (!engine->eop_found); + + if(enable_credit_mp) + engine->eop_found = 0; + + return rc_len; +} + +static void sgt_free_with_pages(struct sg_table *sgt, int dir, + struct pci_dev *pdev) +{ + struct scatterlist *sg = sgt->sgl; + int npages = sgt->orig_nents; + int i; + + for (i = 0; i < npages; i++, sg = sg_next(sg)) { + struct page *pg = sg_page(sg); + dma_addr_t bus = sg_dma_address(sg); + + if (pg) { + if (pdev) + pci_unmap_page(pdev, bus, PAGE_SIZE, dir); + __free_page(pg); + } else + break; + } + sg_free_table(sgt); + memset(sgt, 0, sizeof(struct sg_table)); +} + +static int sgt_alloc_with_pages(struct sg_table *sgt, unsigned int npages, + int dir, struct pci_dev *pdev) +{ + struct scatterlist *sg; + int i; + + if (sg_alloc_table(sgt, npages, GFP_KERNEL)) { + pr_info("sgt OOM.\n"); + return -ENOMEM; + } + + sg = sgt->sgl; + for (i = 0; i < npages; i++, sg = sg_next(sg)) { + struct page *pg = alloc_page(GFP_KERNEL); + + if (!pg) { + pr_info("%d/%u, page OOM.\n", i, npages); + goto err_out; + } + + if (pdev) { + dma_addr_t bus = pci_map_page(pdev, pg, 0, PAGE_SIZE, + dir); + if (unlikely(pci_dma_mapping_error(pdev, bus))) { + pr_info("%d/%u, page 0x%p map err.\n", + i, npages, pg); + __free_page(pg); + goto err_out; + } + sg_dma_address(sg) = bus; + sg_dma_len(sg) = PAGE_SIZE; + } + sg_set_page(sg, pg, PAGE_SIZE, 0); + } + + sgt->orig_nents = sgt->nents = npages; + + return 0; + +err_out: + sgt_free_with_pages(sgt, dir, pdev); + return -ENOMEM; +} + +/* + * !NOTE! reference/demo purpose only + * xdma_cyclic_transfer_setup is used for streaming C2H transfers: + * - A list of buffers are pre-allocated for incoming streaming data + * - the ring of the buffers is allowed to wrap around + */ +int xdma_cyclic_transfer_setup(struct xdma_engine *engine) +{ + struct xdma_dev *xdev; + struct xdma_transfer *xfer; + dma_addr_t bus; + unsigned long flags; + int i; + int rc; + + BUG_ON(!engine); + xdev = engine->xdev; + BUG_ON(!xdev); + + if (engine->cyclic_req) { + pr_info("%s: exclusive access already taken.\n", + engine->name); + return -EBUSY; + } + + spin_lock_irqsave(&engine->lock, flags); + + engine->rx_tail = 0; + engine->rx_head = 0; + engine->rx_overrun = 0; + engine->eop_found = 0; + + rc = sgt_alloc_with_pages(&engine->cyclic_sgt, CYCLIC_RX_PAGES_MAX, + engine->dir, xdev->pdev); + if (rc < 0) { + pr_info("%s cyclic pages %u OOM.\n", + engine->name, CYCLIC_RX_PAGES_MAX); + goto err_out; + } + + engine->cyclic_req = xdma_init_request(&engine->cyclic_sgt, 0); + if (!engine->cyclic_req) { + pr_info("%s cyclic request OOM.\n", engine->name); + rc = -ENOMEM; + goto err_out; + } + +#ifdef __LIBXDMA_DEBUG__ + xdma_request_cb_dump(engine->cyclic_req); +#endif + + rc = transfer_init(engine, engine->cyclic_req); + if (rc < 0) + goto err_out; + + xfer = &engine->cyclic_req->xfer; + + /* replace source addresses with result write-back addresses */ + memset(engine->cyclic_result, 0, + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result)); + bus = engine->cyclic_result_bus; + for (i = 0; i < xfer->desc_num; i++) { + xfer->desc_virt[i].src_addr_lo = cpu_to_le32(PCI_DMA_L(bus)); + xfer->desc_virt[i].src_addr_hi = cpu_to_le32(PCI_DMA_H(bus)); + bus += sizeof(struct xdma_result); + } + /* set control of all descriptors */ + for (i = 0; i < xfer->desc_num; i++) { + xdma_desc_control_clear(xfer->desc_virt + i, LS_BYTE_MASK); + xdma_desc_control_set(xfer->desc_virt + i, + XDMA_DESC_EOP | XDMA_DESC_COMPLETED); + } + + /* make this a cyclic transfer */ + xdma_transfer_cyclic(xfer); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); +#endif + + if(enable_credit_mp){ + //write_register(RX_BUF_PAGES,&engine->sgdma_regs->credits); + write_register(128, &engine->sgdma_regs->credits, 0); + } + + spin_unlock_irqrestore(&engine->lock, flags); + + /* start cyclic transfer */ + transfer_queue(engine, xfer); + + return 0; + + /* unwind on errors */ +err_out: + if (engine->cyclic_req) { + xdma_request_free(engine->cyclic_req); + engine->cyclic_req = NULL; + } + + if (engine->cyclic_sgt.orig_nents) { + sgt_free_with_pages(&engine->cyclic_sgt, engine->dir, + xdev->pdev); + engine->cyclic_sgt.orig_nents = 0; + engine->cyclic_sgt.nents = 0; + engine->cyclic_sgt.sgl = NULL; + } + + spin_unlock_irqrestore(&engine->lock, flags); + + return rc; +} + + +static int cyclic_shutdown_polled(struct xdma_engine *engine) +{ + BUG_ON(!engine); + + spin_lock(&engine->lock); + + dbg_tfr("Polling for shutdown completion\n"); + do { + engine_status_read(engine, 1, 0); + schedule(); + } while (engine->status & XDMA_STAT_BUSY); + + if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { + dbg_tfr("Engine has stopped\n"); + + if (!list_empty(&engine->transfer_list)) + engine_transfer_dequeue(engine); + + engine_service_shutdown(engine); + } + + dbg_tfr("Shutdown completion polling done\n"); + spin_unlock(&engine->lock); + + return 0; +} + +static int cyclic_shutdown_interrupt(struct xdma_engine *engine) +{ + int rc; + + BUG_ON(!engine); + + rc = wait_event_interruptible_timeout(engine->shutdown_wq, + !engine->running, msecs_to_jiffies(10000)); + +#if 0 + if (rc) { + dbg_tfr("wait_event_interruptible=%d\n", rc); + return rc; + } +#endif + + if (engine->running) { + pr_info("%s still running?!, %d\n", engine->name, rc); + return -EINVAL; + } + + return rc; +} + +int xdma_cyclic_transfer_teardown(struct xdma_engine *engine) +{ + int rc; + struct xdma_dev *xdev = engine->xdev; + struct xdma_transfer *transfer; + unsigned long flags; + + transfer = engine_cyclic_stop(engine); + + spin_lock_irqsave(&engine->lock, flags); + if (transfer) { + dbg_tfr("%s: stop transfer 0x%p.\n", engine->name, transfer); + if (transfer != &engine->cyclic_req->xfer) { + pr_info("%s unexpected transfer 0x%p/0x%p\n", + engine->name, transfer, + &engine->cyclic_req->xfer); + } + } + /* allow engine to be serviced after stop request */ + spin_unlock_irqrestore(&engine->lock, flags); + + /* wait for engine to be no longer running */ + if (poll_mode) + rc = cyclic_shutdown_polled(engine); + else + rc = cyclic_shutdown_interrupt(engine); + + /* obtain spin lock to atomically remove resources */ + spin_lock_irqsave(&engine->lock, flags); + + if (engine->cyclic_req) { + xdma_request_free(engine->cyclic_req); + engine->cyclic_req = NULL; + } + + if (engine->cyclic_sgt.orig_nents) { + sgt_free_with_pages(&engine->cyclic_sgt, engine->dir, + xdev->pdev); + engine->cyclic_sgt.orig_nents = 0; + engine->cyclic_sgt.nents = 0; + engine->cyclic_sgt.sgl = NULL; + } + + spin_unlock_irqrestore(&engine->lock, flags); + + return 0; +} + +int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg) +{ + int rv; + unsigned long dst; + u32 w = XDMA_CTRL_NON_INCR_ADDR; + + dbg_perf("IOCTL_XDMA_ADDRMODE_SET\n"); + rv = get_user(dst, (int __user *)arg); + + if (rv == 0) { + engine->non_incr_addr = !!dst; + if (engine->non_incr_addr) + write_register(w, &engine->regs->control_w1s, + (unsigned long)(&engine->regs->control_w1s) - + (unsigned long)(&engine->regs)); + else + write_register(w, &engine->regs->control_w1c, + (unsigned long)(&engine->regs->control_w1c) - + (unsigned long)(&engine->regs)); + } + engine_alignments(engine); + + return rv; +} + diff --git a/sources/xdma_driver/libxdma/libxdma.h b/sources/xdma_driver/libxdma/libxdma.h new file mode 100644 index 0000000..bbf1a45 --- /dev/null +++ b/sources/xdma_driver/libxdma/libxdma.h @@ -0,0 +1,601 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef XDMA_LIB_H +#define XDMA_LIB_H + +#include <linux/version.h> +#include <linux/types.h> +#include <linux/uaccess.h> +#include <linux/module.h> +#include <linux/dma-mapping.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/jiffies.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/workqueue.h> + +/* Switch debug printing on/off */ +#define XDMA_DEBUG 0 + +/* SECTION: Preprocessor macros/constants */ +#define XDMA_BAR_NUM (6) + +/* maximum amount of register space to map */ +#define XDMA_BAR_SIZE (0x8000UL) + +/* Use this definition to poll several times between calls to schedule */ +#define NUM_POLLS_PER_SCHED 100 + +#define XDMA_CHANNEL_NUM_MAX (4) +/* + * interrupts per engine, rad2_vul.sv:237 + * .REG_IRQ_OUT (reg_irq_from_ch[(channel*2) +: 2]), + */ +#define XDMA_ENG_IRQ_NUM (1) +#define MAX_EXTRA_ADJ (15) +#define RX_STATUS_EOP (1) + +/* Target internal components on XDMA control BAR */ +#define XDMA_OFS_INT_CTRL (0x2000UL) +#define XDMA_OFS_CONFIG (0x3000UL) + +/* maximum number of desc per transfer request */ +#define XDMA_TRANSFER_MAX_DESC (2048) + +/* maximum size of a single DMA transfer descriptor */ +#define XDMA_DESC_BLEN_BITS 28 +#define XDMA_DESC_BLEN_MAX ((1 << (XDMA_DESC_BLEN_BITS)) - 1) + +/* bits of the SG DMA control register */ +#define XDMA_CTRL_RUN_STOP (1UL << 0) +#define XDMA_CTRL_IE_DESC_STOPPED (1UL << 1) +#define XDMA_CTRL_IE_DESC_COMPLETED (1UL << 2) +#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH (1UL << 3) +#define XDMA_CTRL_IE_MAGIC_STOPPED (1UL << 4) +#define XDMA_CTRL_IE_IDLE_STOPPED (1UL << 6) +#define XDMA_CTRL_IE_READ_ERROR (0x1FUL << 9) +#define XDMA_CTRL_IE_DESC_ERROR (0x1FUL << 19) +#define XDMA_CTRL_NON_INCR_ADDR (1UL << 25) +#define XDMA_CTRL_POLL_MODE_WB (1UL << 26) + +/* bits of the SG DMA status register */ +#define XDMA_STAT_BUSY (1UL << 0) +#define XDMA_STAT_DESC_STOPPED (1UL << 1) +#define XDMA_STAT_DESC_COMPLETED (1UL << 2) +#define XDMA_STAT_ALIGN_MISMATCH (1UL << 3) +#define XDMA_STAT_MAGIC_STOPPED (1UL << 4) +#define XDMA_STAT_INVALID_LEN (1UL << 5) +#define XDMA_STAT_IDLE_STOPPED (1UL << 6) + +#define XDMA_STAT_COMMON_ERR_MASK \ + (XDMA_STAT_ALIGN_MISMATCH | XDMA_STAT_MAGIC_STOPPED | \ + XDMA_STAT_INVALID_LEN) + +/* desc_error, C2H & H2C */ +#define XDMA_STAT_DESC_UNSUPP_REQ (1UL << 19) +#define XDMA_STAT_DESC_COMPL_ABORT (1UL << 20) +#define XDMA_STAT_DESC_PARITY_ERR (1UL << 21) +#define XDMA_STAT_DESC_HEADER_EP (1UL << 22) +#define XDMA_STAT_DESC_UNEXP_COMPL (1UL << 23) + +#define XDMA_STAT_DESC_ERR_MASK \ + (XDMA_STAT_DESC_UNSUPP_REQ | XDMA_STAT_DESC_COMPL_ABORT | \ + XDMA_STAT_DESC_PARITY_ERR | XDMA_STAT_DESC_HEADER_EP | \ + XDMA_STAT_DESC_UNEXP_COMPL) + +/* read error: H2C */ +#define XDMA_STAT_H2C_R_UNSUPP_REQ (1UL << 9) +#define XDMA_STAT_H2C_R_COMPL_ABORT (1UL << 10) +#define XDMA_STAT_H2C_R_PARITY_ERR (1UL << 11) +#define XDMA_STAT_H2C_R_HEADER_EP (1UL << 12) +#define XDMA_STAT_H2C_R_UNEXP_COMPL (1UL << 13) + +#define XDMA_STAT_H2C_R_ERR_MASK \ + (XDMA_STAT_H2C_R_UNSUPP_REQ | XDMA_STAT_H2C_R_COMPL_ABORT | \ + XDMA_STAT_H2C_R_PARITY_ERR | XDMA_STAT_H2C_R_HEADER_EP | \ + XDMA_STAT_H2C_R_UNEXP_COMPL) + +/* write error, H2C only */ +#define XDMA_STAT_H2C_W_DECODE_ERR (1UL << 14) +#define XDMA_STAT_H2C_W_SLAVE_ERR (1UL << 15) + +#define XDMA_STAT_H2C_W_ERR_MASK \ + (XDMA_STAT_H2C_W_DECODE_ERR | XDMA_STAT_H2C_W_SLAVE_ERR) + +/* read error: C2H */ +#define XDMA_STAT_C2H_R_DECODE_ERR (1UL << 9) +#define XDMA_STAT_C2H_R_SLAVE_ERR (1UL << 10) + +#define XDMA_STAT_C2H_R_ERR_MASK \ + (XDMA_STAT_C2H_R_DECODE_ERR | XDMA_STAT_C2H_R_SLAVE_ERR) + +/* all combined */ +#define XDMA_STAT_H2C_ERR_MASK \ + (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \ + XDMA_STAT_H2C_R_ERR_MASK | XDMA_STAT_H2C_W_ERR_MASK) + +#define XDMA_STAT_C2H_ERR_MASK \ + (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \ + XDMA_STAT_C2H_R_ERR_MASK) + +/* bits of the SGDMA descriptor control field */ +#define XDMA_DESC_STOPPED (1UL << 0) +#define XDMA_DESC_COMPLETED (1UL << 1) +#define XDMA_DESC_EOP (1UL << 4) + +#define XDMA_PERF_RUN (1UL << 0) +#define XDMA_PERF_CLEAR (1UL << 1) +#define XDMA_PERF_AUTO (1UL << 2) + +#define MAGIC_ENGINE 0xEEEEEEEEUL +#define MAGIC_DEVICE 0xDDDDDDDDUL + +/* upper 16-bits of engine identifier register */ +#define XDMA_ID_H2C 0x1fc0U +#define XDMA_ID_C2H 0x1fc1U + +/* for C2H AXI-ST mode */ +#define CYCLIC_RX_PAGES_MAX 256 + +#define LS_BYTE_MASK 0x000000FFUL + +#define BLOCK_ID_MASK 0xFFF00000 +#define BLOCK_ID_HEAD 0x1FC00000 + +#define IRQ_BLOCK_ID 0x1fc20000UL +#define CONFIG_BLOCK_ID 0x1fc30000UL + +#define WB_COUNT_MASK 0x00ffffffUL +#define WB_ERR_MASK (1UL << 31) +#define POLL_TIMEOUT_SECONDS 10 + +#define MAX_USER_IRQ 16 + +#define MAX_DESC_BUS_ADDR (0xffffffffULL) + +#define DESC_MAGIC 0xAD4B0000UL + +#define C2H_WB 0x52B4UL + +#define MAX_NUM_ENGINES (XDMA_CHANNEL_NUM_MAX * 2) +#define H2C_CHANNEL_OFFSET 0x1000 +#define SGDMA_OFFSET_FROM_CHANNEL 0x4000 +#define CHANNEL_SPACING 0x100 +#define TARGET_SPACING 0x1000 + +#define BYPASS_MODE_SPACING 0x0100 + +/* obtain the 32 most significant (high) bits of a 32-bit or 64-bit address */ +#define PCI_DMA_H(addr) ((addr >> 16) >> 16) +/* obtain the 32 least significant (low) bits of a 32-bit or 64-bit address */ +#define PCI_DMA_L(addr) (addr & 0xffffffffUL) + +#ifndef VM_RESERVED + #define VMEM_FLAGS (VM_IO | VM_DONTEXPAND | VM_DONTDUMP) +#else + #define VMEM_FLAGS (VM_IO | VM_RESERVED) +#endif + +#ifdef __LIBXDMA_DEBUG__ +#define dbg_io pr_err +#define dbg_fops pr_err +#define dbg_perf pr_err +#define dbg_sg pr_err +#define dbg_tfr pr_err +#define dbg_irq pr_err +#define dbg_init pr_err +#define dbg_desc pr_err +#else +/* disable debugging */ +#define dbg_io(...) +#define dbg_fops(...) +#define dbg_perf(...) +#define dbg_sg(...) +#define dbg_tfr(...) +#define dbg_irq(...) +#define dbg_init(...) +#define dbg_desc(...) +#endif + +/* SECTION: Enum definitions */ +enum transfer_state { + TRANSFER_STATE_NEW = 0, + TRANSFER_STATE_SUBMITTED, + TRANSFER_STATE_COMPLETED, + TRANSFER_STATE_FAILED, + TRANSFER_STATE_ABORTED +}; + +enum shutdown_state { + ENGINE_SHUTDOWN_NONE = 0, /* No shutdown in progress */ + ENGINE_SHUTDOWN_REQUEST = 1, /* engine requested to shutdown */ + ENGINE_SHUTDOWN_IDLE = 2 /* engine has shutdown and is idle */ +}; + +enum dev_capabilities { + CAP_64BIT_DMA = 2, + CAP_64BIT_DESC = 4, + CAP_ENGINE_WRITE = 8, + CAP_ENGINE_READ = 16 +}; + +/* SECTION: Structure definitions */ + +struct config_regs { + u32 identifier; + u32 reserved_1[4]; + u32 msi_enable; +}; + +/** + * SG DMA Controller status and control registers + * + * These registers make the control interface for DMA transfers. + * + * It sits in End Point (FPGA) memory BAR[0] for 32-bit or BAR[0:1] for 64-bit. + * It references the first descriptor which exists in Root Complex (PC) memory. + * + * @note The registers must be accessed using 32-bit (PCI DWORD) read/writes, + * and their values are in little-endian byte ordering. + */ +struct engine_regs { + u32 identifier; + u32 control; + u32 control_w1s; + u32 control_w1c; + u32 reserved_1[12]; /* padding */ + + u32 status; + u32 status_rc; + u32 completed_desc_count; + u32 alignments; + u32 reserved_2[14]; /* padding */ + + u32 poll_mode_wb_lo; + u32 poll_mode_wb_hi; + u32 interrupt_enable_mask; + u32 interrupt_enable_mask_w1s; + u32 interrupt_enable_mask_w1c; + u32 reserved_3[9]; /* padding */ + + u32 perf_ctrl; + u32 perf_cyc_lo; + u32 perf_cyc_hi; + u32 perf_dat_lo; + u32 perf_dat_hi; + u32 perf_pnd_lo; + u32 perf_pnd_hi; +} __packed; + +struct engine_sgdma_regs { + u32 identifier; + u32 reserved_1[31]; /* padding */ + + /* bus address to first descriptor in Root Complex Memory */ + u32 first_desc_lo; + u32 first_desc_hi; + /* number of adjacent descriptors at first_desc */ + u32 first_desc_adjacent; + u32 credits; +} __packed; + +struct msix_vec_table_entry { + u32 msi_vec_addr_lo; + u32 msi_vec_addr_hi; + u32 msi_vec_data_lo; + u32 msi_vec_data_hi; +} __packed; + +struct msix_vec_table { + struct msix_vec_table_entry entry_list[32]; +} __packed; + +struct interrupt_regs { + u32 identifier; + u32 user_int_enable; + u32 user_int_enable_w1s; + u32 user_int_enable_w1c; + u32 channel_int_enable; + u32 channel_int_enable_w1s; + u32 channel_int_enable_w1c; + u32 reserved_1[9]; /* padding */ + + u32 user_int_request; + u32 channel_int_request; + u32 user_int_pending; + u32 channel_int_pending; + u32 reserved_2[12]; /* padding */ + + u32 user_msi_vector[8]; + u32 channel_msi_vector[8]; +} __packed; + +struct sgdma_common_regs { + u32 padding[8]; + u32 credit_mode_enable; + u32 credit_mode_enable_w1s; + u32 credit_mode_enable_w1c; +} __packed; + + +/* Structure for polled mode descriptor writeback */ +struct xdma_poll_wb { + u32 completed_desc_count; + u32 reserved_1[7]; +} __packed; + + +/** + * Descriptor for a single contiguous memory block transfer. + * + * Multiple descriptors are linked by means of the next pointer. An additional + * extra adjacent number gives the amount of extra contiguous descriptors. + * + * The descriptors are in root complex memory, and the bytes in the 32-bit + * words must be in little-endian byte ordering. + */ +struct xdma_desc { + u32 control; + u32 bytes; /* transfer length in bytes */ + u32 src_addr_lo; /* source address (low 32-bit) */ + u32 src_addr_hi; /* source address (high 32-bit) */ + u32 dst_addr_lo; /* destination address (low 32-bit) */ + u32 dst_addr_hi; /* destination address (high 32-bit) */ + /* + * next descriptor in the single-linked list of descriptors; + * this is the PCIe (bus) address of the next descriptor in the + * root complex memory + */ + u32 next_lo; /* next desc address (low 32-bit) */ + u32 next_hi; /* next desc address (high 32-bit) */ +} __packed; + +/* 32 bytes (four 32-bit words) or 64 bytes (eight 32-bit words) */ +struct xdma_result { + u32 status; + u32 length; + u32 reserved_1[6]; /* padding */ +} __packed; + +struct sw_desc { + dma_addr_t addr; + unsigned int len; +}; + +/* Describes a (SG DMA) single transfer for the engine */ +struct xdma_transfer { + struct list_head entry; /* queue of non-completed transfers */ + struct xdma_desc *desc_virt; /* virt addr of the 1st descriptor */ + dma_addr_t desc_bus; /* bus addr of the first descriptor */ + int desc_adjacent; /* adjacent descriptors at desc_bus */ + int desc_num; /* number of descriptors in transfer */ + enum dma_data_direction dir; + wait_queue_head_t wq; /* wait queue for transfer completion */ + + enum transfer_state state; /* state of the transfer */ + unsigned int flags; +#define XFER_FLAG_NEED_UNMAP 0x1 + int cyclic; /* flag if transfer is cyclic */ + int last_in_request; /* flag if last within request */ + unsigned int len; + struct sg_table *sgt; +}; + +struct xdma_request_cb { + struct sg_table *sgt; + unsigned int total_len; + u64 ep_addr; + + struct xdma_transfer xfer; + + unsigned int sw_desc_idx; + unsigned int sw_desc_cnt; + struct sw_desc sdesc[0]; +}; + +struct xdma_engine { + unsigned long magic; /* structure ID for sanity checks */ + struct xdma_dev *xdev; /* parent device */ + char name[5]; /* name of this engine */ + int version; /* version of this engine */ + //dev_t cdevno; /* character device major:minor */ + //struct cdev cdev; /* character device (embedded struct) */ + + /* HW register address offsets */ + struct engine_regs *regs; /* Control reg BAR offset */ + struct engine_sgdma_regs *sgdma_regs; /* SGDAM reg BAR offset */ + u32 bypass_offset; /* Bypass mode BAR offset */ + + /* Engine state, configuration and flags */ + enum shutdown_state shutdown; /* engine shutdown mode */ + enum dma_data_direction dir; + int device_open; /* flag if engine node open, ST mode only */ + int running; /* flag if the driver started engine */ + int non_incr_addr; /* flag if non-incremental addressing used */ + int streaming; + int addr_align; /* source/dest alignment in bytes */ + int len_granularity; /* transfer length multiple */ + int addr_bits; /* HW datapath address width */ + int channel; /* engine indices */ + int max_extra_adj; /* descriptor prefetch capability */ + int desc_dequeued; /* num descriptors of completed transfers */ + u32 status; /* last known status of device */ + u32 interrupt_enable_mask_value;/* only used for MSIX mode to store per-engine interrupt mask value */ + + /* Transfer list management */ + struct list_head transfer_list; /* queue of transfers */ + + /* Members applicable to AXI-ST C2H (cyclic) transfers */ + struct xdma_result *cyclic_result; + dma_addr_t cyclic_result_bus; /* bus addr for transfer */ + struct xdma_request_cb *cyclic_req; + struct sg_table cyclic_sgt; + u8 eop_found; /* used only for cyclic(rx:c2h) */ + + int rx_tail; /* follows the HW */ + int rx_head; /* where the SW reads from */ + int rx_overrun; /* flag if overrun occured */ + + /* for copy from cyclic buffer to user buffer */ + unsigned int user_buffer_index; + + /* Members associated with polled mode support */ + u8 *poll_mode_addr_virt; /* virt addr for descriptor writeback */ + dma_addr_t poll_mode_bus; /* bus addr for descriptor writeback */ + + /* Members associated with interrupt mode support */ + wait_queue_head_t shutdown_wq; /* wait queue for shutdown sync */ + spinlock_t lock; /* protects concurrent access */ + int prev_cpu; /* remember CPU# of (last) locker */ + int msix_irq_line; /* MSI-X vector for this engine */ + u32 irq_bitmask; /* IRQ bit mask for this engine */ + struct work_struct work; /* Work queue for interrupt handling */ + + spinlock_t desc_lock; /* protects concurrent access */ + dma_addr_t desc_bus; + struct xdma_desc *desc; + + /* for performance test support */ + struct xdma_performance_ioctl *xdma_perf; /* perf test control */ + wait_queue_head_t xdma_perf_wq; /* Perf test sync */ +}; + +struct xdma_user_irq { + struct xdma_dev *xdev; /* parent device */ + u8 user_idx; /* 0 ~ 15 */ + u8 events_irq; /* accumulated IRQs */ + spinlock_t events_lock; /* lock to safely update events_irq */ + wait_queue_head_t events_wq; /* wait queue to sync waiting threads */ + irq_handler_t handler; + + void *dev; +}; + +/* XDMA PCIe device specific book-keeping */ +#define XDEV_FLAG_OFFLINE 0x1 +struct xdma_dev { + struct list_head list_head; + struct list_head rcu_node; + + unsigned long magic; /* structure ID for sanity checks */ + struct pci_dev *pdev; /* pci device struct from probe() */ + int idx; /* dev index */ + + const char *mod_name; /* name of module owning the dev */ + + spinlock_t lock; /* protects concurrent access */ + unsigned int flags; + + /* PCIe BAR management */ + void *__iomem bar[XDMA_BAR_NUM]; /* addresses for mapped BARs */ + int user_bar_idx; /* BAR index of user logic */ + int config_bar_idx; /* BAR index of XDMA config logic */ + int bypass_bar_idx; /* BAR index of XDMA bypass logic */ + int regions_in_use; /* flag if dev was in use during probe() */ + int got_regions; /* flag if probe() obtained the regions */ + + int user_max; + int c2h_channel_max; + int h2c_channel_max; + + /* Interrupt management */ + int irq_count; /* interrupt counter */ + int irq_line; /* flag if irq allocated successfully */ + int msi_enabled; /* flag if msi was enabled for the device */ + int msix_enabled; /* flag if msi-x was enabled for the device */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,12,0) + struct msix_entry entry[32]; /* msi-x vector/entry table */ +#endif + struct xdma_user_irq user_irq[16]; /* user IRQ management */ + unsigned int mask_irq_user; + + /* XDMA engine management */ + int engines_num; /* Total engine count */ + u32 mask_irq_h2c; + u32 mask_irq_c2h; + struct xdma_engine engine_h2c[XDMA_CHANNEL_NUM_MAX]; + struct xdma_engine engine_c2h[XDMA_CHANNEL_NUM_MAX]; + + /* SD_Accel specific */ + enum dev_capabilities capabilities; + u64 feature_id; +}; + +static inline int xdma_device_flag_check(struct xdma_dev *xdev, unsigned int f) +{ + unsigned long flags; + + spin_lock_irqsave(&xdev->lock, flags); + if (xdev->flags & f) { + spin_unlock_irqrestore(&xdev->lock, flags); + return 1; + } + spin_unlock_irqrestore(&xdev->lock, flags); + return 0; +} + +static inline int xdma_device_flag_test_n_set(struct xdma_dev *xdev, + unsigned int f) +{ + unsigned long flags; + int rv = 0; + + spin_lock_irqsave(&xdev->lock, flags); + if (xdev->flags & f) { + spin_unlock_irqrestore(&xdev->lock, flags); + rv = 1; + } else + xdev->flags |= f; + spin_unlock_irqrestore(&xdev->lock, flags); + return rv; +} + +static inline void xdma_device_flag_set(struct xdma_dev *xdev, unsigned int f) +{ + unsigned long flags; + + spin_lock_irqsave(&xdev->lock, flags); + xdev->flags |= f; + spin_unlock_irqrestore(&xdev->lock, flags); +} + +static inline void xdma_device_flag_clear(struct xdma_dev *xdev, unsigned int f) +{ + unsigned long flags; + + spin_lock_irqsave(&xdev->lock, flags); + xdev->flags &= ~f; + spin_unlock_irqrestore(&xdev->lock, flags); +} + +void write_register(u32 value, void *iomem); +u32 read_register(void *iomem); + +struct xdma_dev *xdev_find_by_pdev(struct pci_dev *pdev); + +void xdma_device_offline(struct pci_dev *pdev, void *dev_handle); +void xdma_device_online(struct pci_dev *pdev, void *dev_handle); + +int xdma_performance_submit(struct xdma_dev *xdev, struct xdma_engine *engine); +struct xdma_transfer *engine_cyclic_stop(struct xdma_engine *engine); +void enable_perf(struct xdma_engine *engine); +void get_perf_stats(struct xdma_engine *engine); + +int xdma_cyclic_transfer_setup(struct xdma_engine *engine); +int xdma_cyclic_transfer_teardown(struct xdma_engine *engine); +ssize_t xdma_engine_read_cyclic(struct xdma_engine *, char __user *, size_t, + int); +int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg); + +#endif /* XDMA_LIB_H */ diff --git a/sources/xdma_driver/libxdma/version.h b/sources/xdma_driver/libxdma/version.h new file mode 100644 index 0000000..703424d --- /dev/null +++ b/sources/xdma_driver/libxdma/version.h @@ -0,0 +1,28 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef __XDMA_VERSION_H__ +#define __XDMA_VERSION_H__ + +#define DRV_MOD_MAJOR 2017 +#define DRV_MOD_MINOR 1 +#define DRV_MOD_PATCHLEVEL 36 + +#define DRV_MODULE_VERSION \ + __stringify(DRV_MOD_MAJOR) "." \ + __stringify(DRV_MOD_MINOR) "." \ + __stringify(DRV_MOD_PATCHLEVEL) + +#define DRV_MOD_VERSION_NUMBER \ + ((DRV_MOD_MAJOR)*1000 + (DRV_MOD_MINOR)*100 + DRV_MOD_PATCHLEVEL) + +#endif /* ifndef __XDMA_VERSION_H__ */ diff --git a/sources/xdma_driver/load_driver.sh b/sources/xdma_driver/load_driver.sh new file mode 100755 index 0000000..9910136 --- /dev/null +++ b/sources/xdma_driver/load_driver.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Make sure only root can run our script +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" 1>&2 + exit 1 +fi + +cd xdma +make clean +make -j +if [ ! $? == 0 ]; then + echo "Error: Kernel module did not compile properly." + echo " FAILED" + exit 1 +fi +cd .. + + +# Remove the existing xdma kernel module +lsmod | grep xdma +if [ $? -eq 0 ]; then + rmmod xdma +fi +echo -n "Loading xdma driver..." +# Use the following command to Load the driver in the default +# or interrupt drive mode. This will allow the driver to use +# interrupts to signal when DMA transfers are completed. +insmod xdma/xdma.ko enable_credit_mp=1 +# Use the following command to Load the driver in Polling +# mode rather than than interrupt mode. This will allow the +# driver to use polling to determ when DMA transfers are +# completed. +#insmod ../xdma/xdma.ko poll_mode=1 + +if [ ! $? == 0 ]; then + echo "Error: Kernel module did not load properly." + echo " FAILED" + exit 1 +fi + +# Check to see if the xdma devices were recognized +echo "" +cat /proc/devices | grep xdma > /dev/null +returnVal=$? +if [ $returnVal == 0 ]; then + # Installed devices were recognized. + echo "The Kernel module installed correctly and the xmda devices were recognized." +else + # No devices were installed. + echo "Error: The Kernel module installed correctly, but no devices were recognized." + echo " FAILED" + exit 1 +fi + +echo " DONE" diff --git a/sources/xdma_driver/readme.txt b/sources/xdma_driver/readme.txt new file mode 100644 index 0000000..cbc80f0 --- /dev/null +++ b/sources/xdma_driver/readme.txt @@ -0,0 +1,140 @@ +Modifications to make the dirver compatible with newer Linux kernels: +===================================================================== +For kernel version >= 5.3.0: +- xdma/: + - cdev_ctrl.c: Interface changed for access_ok(): https://forums.xilinx.com/t5/PCIe-and-CPM/PCIe-DMA-driver-compilation-issues-in-Linux-Ubuntu-19-04/td-p/1022239 + + - cdev_xvc.c: + - libxdma.c: Does not need mmiowb() + +(This readme is less relevant as we've modified the file structure) + +The files in this directory provide Xilinx PCIe DMA drivers, example software, +and example test scripts that can be used to exercise the Xilinx PCIe DMA IP. + +This software can be used directly or referenced to create drivers and software +for your Xilinx FPGA hardware design. + +Directory and file description: +=============================== + - xdma/: This directory contains the Xilinx PCIe DMA kernel module + driver files. + + - libxdma/: This directory contains support files for the kernel driver module, + which interfaces directly with the XDMA IP. + + - include/: This directory contains all include files that are needed for + compiling driver. + + - etc/: This directory contains rules for the Xilinx PCIe DMA kernel module + and software. The files in this directory should be copied to the /etc/ + directory on your linux system. + + - tests/: This directory contains example application software to exercise the + provided kernel module driver and Xilinx PCIe DMA IP. This directory + also contains the following scripts and directories. + + - load_driver.sh: + This script loads the kernel module and creates the necissary + kernel nodes used by the provided software. + The The kernel device nodes will be created under /dev/xdma*. + Additional device nodes are created under /dev/xdma/card* to + more easily differentiate between multiple PCIe DMA enabled + cards. Root permissions will be required to run this script. + + - run_test.sh: + This script runs sample tests on a Xilinx PCIe DMA target and + returns a pass (0) or fail (1) result. + This script is intended for use with the PCIe DMA example + design. + + - perform_hwcount.sh: + This script runs hardware performance for XDMA for both Host to + Card (H2C) and Card to Host (C2H). The result are copied to + 'hw_log_h2c.txt' and hw_log_c2h.txt' text files. + For each direction the performance script loops from 64 bytes + to 4MBytes and generate performance numbers (byte size doubles + for each loop count). + You can grep for 'data rate' on those two files to see data + rate values. + Data rate values are in percentage of maximum throughput. + Maximum data rate for x8 Gen3 is 8Gbytes/s, so for a x8Gen3 + design value of 0.81 data rate is 0.81*8 = 6.48Gbytes/s. + Maximum data rate for x16 Gen3 is 16Gbytes/s, so for a x16Gen3 + design value of 0.78 data rate is 0.78*16 = 12.48Gbytes/s. + This program can be run on AXI-MM example design. + AXI-ST example design is a loopback design, both H2C and C2H + are connected. Running on AXI-ST example design will not + generate proper numbers. + If a AXI-ST design is independent of H2C and C2H, performance + number can be generated. + - data/: + This directory contains binary data files that are used for DMA + data transfers to the Xilinx FPGA PCIe endpoint device. + +Usage: + - Change directory to the driver directory. + cd xdma + - Compile and install the kernel module driver. + make install + - Change directory to the tools directory. + cd tools + - Compile the provided example test tools. + make + - Copy the provided driver rules from the etc directory to the /etc/ directory + on your system. + cp ../etc/udev/rules.d/* /etc/udev/rules.d/ + - Load the kernel module driver: + a. modprobe xdma + b. using the provided script. + cd tests + ./load_driver.sh + - Run the provided test script to generate basic DMA traffic. + ./run_test.sh + - Check driver Version number + modinfo xdma (or) + modinfo ../xdma/xdma.ko + +Updates and Backward Compaitiblity: + - The following features were added to the PCIe DMA IP and driver in Vivado + 2016.1. These features cannot be used with PCIe DMA IP if the IP was + generated using a Vivado build earlier than 2016.1. + - Poll Mode: Earlier versions of Vivado only support interrupt mode which + is the default behavior of the driver. + - Source/Destination Address: Earlier versions of Vivado PCIe DMA IP + required the low-order bits of the Source and Destination address to be + the same. + As of 2016.1 this restriction has been removed and the Source and + Destination addresses can be any arbitrary address that is valid for + your system. + +Frequently asked questions: + Q: How do I uninstall the kernel module driver? + A: Use the following commands to uninstall the driver. + - Uninstall the kernel module. + rmmod -s xdma + - Delete the dma rules that were added. + rm -f /etc/udev/rules.d/60-xdma.rules + rm -f /etc/udev/rules.d/xdma-udev-command.sh + + Q: How do I modify the PCIe Device IDs recognized by the kernel module driver? + A: The xdma/xdma_mod.c file constains the pci_device_id struct that identifies + the PCIe Device IDs that are recognized by the driver in the following + format: + { PCI_DEVICE(0x10ee, 0x8038), }, + Add, remove, or modify the PCIe Device IDs in this struct as desired. Then + uninstall the existing xdma kernel module, compile the driver again, and + re-install the driver using the load_driver.sh script. + + Q: By default the driver uses interupts to signal when DMA transfers are + completed. How do I modify the driver to use polling rather than + interrupts to determine when DMA transactions are completed? + A: The driver can be changed from being interrupt driven (default) to being + polling driven (poll mode) when the kernel module is inserted. To do this + modify the load_driver.sh file as follows: + Change: insmod xdma/xdma.ko + To: insmod xdma/xdma.ko poll_mode=1 + Note: Interrupt vs Poll mode will apply to all DMA channels. If desired the + driver can be modified such that some channels are interrupt driven while + others are polling driven. Refer to the poll mode section of PG195 for + additional information on using the PCIe DMA IP in poll mode. diff --git a/sources/xdma_driver/xdma/Makefile b/sources/xdma_driver/xdma/Makefile new file mode 100644 index 0000000..d1c07a8 --- /dev/null +++ b/sources/xdma_driver/xdma/Makefile @@ -0,0 +1,35 @@ +SHELL = /bin/bash +ifneq ($(xvc_bar_num),) + XVC_FLAGS += -D__XVC_BAR_NUM__=$(xvc_bar_num) +endif + +ifneq ($(xvc_bar_offset),) + XVC_FLAGS += -D__XVC_BAR_OFFSET__=$(xvc_bar_offset) +endif + +$(warning XVC_FLAGS: $(XVC_FLAGS).) + +topdir := $(shell cd $(src)/.. && pwd) + +TARGET_MODULE:=xdma + +EXTRA_CFLAGS := -I$(topdir)/include $(XVC_FLAGS) +#EXTRA_CFLAGS += -D__LIBXDMA_DEBUG__ +#EXTRA_CFLAGS += -DINTERNAL_TESTING + +ifneq ($(KERNELRELEASE),) + $(TARGET_MODULE)-objs := libxdma.o xdma_cdev.o cdev_ctrl.o cdev_events.o cdev_sgdma.o cdev_xvc.o cdev_bypass.o xdma_mod.o + obj-m := $(TARGET_MODULE).o +else + BUILDSYSTEM_DIR:=/lib/modules/$(shell uname -r)/build + PWD:=$(shell pwd) +all : + $(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) modules + +clean: + $(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) clean + +install: all + $(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) modules_install + +endif diff --git a/sources/xdma_driver/xdma/cdev_bypass.c b/sources/xdma_driver/xdma/cdev_bypass.c new file mode 100644 index 0000000..4b84526 --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_bypass.c @@ -0,0 +1,180 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "libxdma_api.h" +#include "xdma_cdev.h" + +#define write_register(v,mem,off) iowrite32(v, mem) + +static int copy_desc_data(struct xdma_transfer *transfer, char __user *buf, + size_t *buf_offset, size_t buf_size) +{ + int i; + int copy_err; + int rc = 0; + + BUG_ON(!buf); + BUG_ON(!buf_offset); + + /* Fill user buffer with descriptor data */ + for (i = 0; i < transfer->desc_num; i++) { + if (*buf_offset + sizeof(struct xdma_desc) <= buf_size) { + copy_err = copy_to_user(&buf[*buf_offset], + transfer->desc_virt + i, + sizeof(struct xdma_desc)); + + if (copy_err) { + dbg_sg("Copy to user buffer failed\n"); + *buf_offset = buf_size; + rc = -EINVAL; + } else { + *buf_offset += sizeof(struct xdma_desc); + } + } else { + rc = -ENOMEM; + } + } + + return rc; +} + +static ssize_t char_bypass_read(struct file *file, char __user *buf, + size_t count, loff_t *pos) +{ + struct xdma_dev *xdev; + struct xdma_engine *engine; + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + struct xdma_transfer *transfer; + struct list_head *idx; + size_t buf_offset = 0; + int rc = 0; + + rc = xcdev_check(__func__, xcdev, 1); + if (rc < 0) + return rc; + xdev = xcdev->xdev; + engine = xcdev->engine; + + dbg_sg("In char_bypass_read()\n"); + + if (count & 3) { + dbg_sg("Buffer size must be a multiple of 4 bytes\n"); + return -EINVAL; + } + + if (!buf) { + dbg_sg("Caught NULL pointer\n"); + return -EINVAL; + } + + if (xdev->bypass_bar_idx < 0) { + dbg_sg("Bypass BAR not present - unsupported operation\n"); + return -ENODEV; + } + + spin_lock(&engine->lock); + + if (!list_empty(&engine->transfer_list)) { + list_for_each(idx, &engine->transfer_list) { + transfer = list_entry(idx, struct xdma_transfer, entry); + + rc = copy_desc_data(transfer, buf, &buf_offset, count); + } + } + + spin_unlock(&engine->lock); + + if (rc < 0) + return rc; + else + return buf_offset; +} + +static ssize_t char_bypass_write(struct file *file, const char __user *buf, + size_t count, loff_t *pos) +{ + struct xdma_dev *xdev; + struct xdma_engine *engine; + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + + u32 desc_data; + u32 *bypass_addr; + size_t buf_offset = 0; + int rc = 0; + int copy_err; + + rc = xcdev_check(__func__, xcdev, 1); + if (rc < 0) + return rc; + xdev = xcdev->xdev; + engine = xcdev->engine; + + if (count & 3) { + dbg_sg("Buffer size must be a multiple of 4 bytes\n"); + return -EINVAL; + } + + if (!buf) { + dbg_sg("Caught NULL pointer\n"); + return -EINVAL; + } + + if (xdev->bypass_bar_idx < 0) { + dbg_sg("Bypass BAR not present - unsupported operation\n"); + return -ENODEV; + } + + dbg_sg("In char_bypass_write()\n"); + + spin_lock(&engine->lock); + + /* Write descriptor data to the bypass BAR */ + bypass_addr = (u32 *)xdev->bar[xdev->bypass_bar_idx]; + bypass_addr += engine->bypass_offset; + while (buf_offset < count) { + copy_err = copy_from_user(&desc_data, &buf[buf_offset], + sizeof(u32)); + if (!copy_err) { + write_register(desc_data, bypass_addr, bypass_addr - engine->bypass_offset); + buf_offset += sizeof(u32); + rc = buf_offset; + } else { + dbg_sg("Error reading data from userspace buffer\n"); + rc = -EINVAL; + break; + } + } + + spin_unlock(&engine->lock); + + + return rc; +} + + +/* + * character device file operations for bypass operation + */ + +static const struct file_operations bypass_fops = { + .owner = THIS_MODULE, + .open = char_open, + .release = char_close, + .read = char_bypass_read, + .write = char_bypass_write, + .mmap = bridge_mmap, +}; + +void cdev_bypass_init(struct xdma_cdev *xcdev) +{ + cdev_init(&xcdev->cdev, &bypass_fops); +} diff --git a/sources/xdma_driver/xdma/cdev_bypass.o.ur-safe b/sources/xdma_driver/xdma/cdev_bypass.o.ur-safe new file mode 100644 index 0000000..0cd6b8c --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_bypass.o.ur-safe @@ -0,0 +1,2 @@ +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/cdev_bypass.o-.text-23c +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/cdev_bypass.o-.text-e9 diff --git a/sources/xdma_driver/xdma/cdev_ctrl.c b/sources/xdma_driver/xdma/cdev_ctrl.c new file mode 100644 index 0000000..6699f5c --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_ctrl.c @@ -0,0 +1,267 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include <linux/ioctl.h> +#include "version.h" +#include "xdma_cdev.h" +#include "cdev_ctrl.h" + +/* + * character device file operations for control bus (through control bridge) + */ +static ssize_t char_ctrl_read(struct file *fp, char __user *buf, size_t count, + loff_t *pos) +{ + struct xdma_cdev *xcdev = (struct xdma_cdev *)fp->private_data; + struct xdma_dev *xdev; + void *reg; + u32 w; + int rv; + + rv = xcdev_check(__func__, xcdev, 0); + if (rv < 0) + return rv; + xdev = xcdev->xdev; + + /* only 32-bit aligned and 32-bit multiples */ + if (*pos & 3) + return -EPROTO; + /* first address is BAR base plus file position offset */ + reg = xdev->bar[xcdev->bar] + *pos; + //w = read_register(reg); + w = ioread32(reg); + dbg_sg("char_ctrl_read(@%p, count=%ld, pos=%d) value = 0x%08x\n", reg, + (long)count, (int)*pos, w); + rv = copy_to_user(buf, &w, 4); + if (rv) + dbg_sg("Copy to userspace failed but continuing\n"); + + *pos += 4; + return 4; +} + +static ssize_t char_ctrl_write(struct file *file, const char __user *buf, + size_t count, loff_t *pos) +{ + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + struct xdma_dev *xdev; + void *reg; + u32 w; + int rv; + + rv = xcdev_check(__func__, xcdev, 0); + if (rv < 0) + return rv; + xdev = xcdev->xdev; + + /* only 32-bit aligned and 32-bit multiples */ + if (*pos & 3) + return -EPROTO; + + /* first address is BAR base plus file position offset */ + reg = xdev->bar[xcdev->bar] + *pos; + rv = copy_from_user(&w, buf, 4); + if (rv) { + pr_info("copy from user failed %d/4, but continuing.\n", rv); + } + + dbg_sg("char_ctrl_write(0x%08x @%p, count=%ld, pos=%d)\n", w, reg, + (long)count, (int)*pos); + //write_register(w, reg); + iowrite32(w, reg); + *pos += 4; + return 4; +} + +static long version_ioctl(struct xdma_cdev *xcdev, void __user *arg) +{ + struct xdma_ioc_info obj; + struct xdma_dev *xdev = xcdev->xdev; + int rv; + + rv = copy_from_user((void *)&obj, arg, sizeof(struct xdma_ioc_info)); + if (rv) { + pr_info("copy from user failed %d/%ld.\n", + rv, sizeof(struct xdma_ioc_info)); + return -EFAULT; + } + memset(&obj, 0, sizeof(obj)); + obj.vendor = xdev->pdev->vendor; + obj.device = xdev->pdev->device; + obj.subsystem_vendor = xdev->pdev->subsystem_vendor; + obj.subsystem_device = xdev->pdev->subsystem_device; + obj.feature_id = xdev->feature_id; + obj.driver_version = DRV_MOD_VERSION_NUMBER; + obj.domain = 0; + obj.bus = PCI_BUS_NUM(xdev->pdev->devfn); + obj.dev = PCI_SLOT(xdev->pdev->devfn); + obj.func = PCI_FUNC(xdev->pdev->devfn); + if (copy_to_user(arg, &obj, sizeof(struct xdma_ioc_info))) + return -EFAULT; + return 0; +} + +long char_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct xdma_cdev *xcdev = (struct xdma_cdev *)filp->private_data; + struct xdma_dev *xdev; + struct xdma_ioc_base ioctl_obj; + long result = 0; + int rv; + + rv = xcdev_check(__func__, xcdev, 0); + if (rv < 0) + return rv; + xdev = xcdev->xdev; + + pr_info("cmd 0x%x, xdev 0x%p, pdev 0x%p.\n", cmd, xdev, xdev->pdev); + + if (_IOC_TYPE(cmd) != XDMA_IOC_MAGIC) { + pr_err("cmd %u, bad magic 0x%x/0x%x.\n", + cmd, _IOC_TYPE(cmd), XDMA_IOC_MAGIC); + return -ENOTTY; + } + + #if LINUX_VERSION_CODE < KERNEL_VERSION(5,3,0) + if (_IOC_DIR(cmd) & _IOC_READ) + result = !access_ok(VERIFY_WRITE, (void __user *)arg, + _IOC_SIZE(cmd)); + else if (_IOC_DIR(cmd) & _IOC_WRITE) + result = !access_ok(VERIFY_READ, (void __user *)arg, + _IOC_SIZE(cmd)); + #else + if (_IOC_DIR(cmd) & _IOC_READ) + result = !access_ok((void __user *)arg, + _IOC_SIZE(cmd)); + else if (_IOC_DIR(cmd) & _IOC_WRITE) + result = !access_ok((void __user *)arg, + _IOC_SIZE(cmd)); + #endif + + if (result) { + pr_err("bad access %ld.\n", result); + return -EFAULT; + } + + switch (cmd) { + case XDMA_IOCINFO: + if (copy_from_user((void *)&ioctl_obj, (void *) arg, + sizeof(struct xdma_ioc_base))) { + pr_err("copy_from_user failed.\n"); + return -EFAULT; + } + + if (ioctl_obj.magic != XDMA_XCL_MAGIC) { + pr_err("magic 0x%x != XDMA_XCL_MAGIC (0x%x).\n", + ioctl_obj.magic, XDMA_XCL_MAGIC); + return -ENOTTY; + } + + return version_ioctl(xcdev, (void __user *)arg); + case XDMA_IOCOFFLINE: + if (!xdev) { + pr_info("cmd %u, xdev NULL.\n", cmd); + return -EINVAL; + } + xdma_device_offline(xdev->pdev, xdev); + break; + case XDMA_IOCONLINE: + if (!xdev) { + pr_info("cmd %u, xdev NULL.\n", cmd); + return -EINVAL; + } + xdma_device_online(xdev->pdev, xdev); + break; + default: + pr_err("UNKNOWN ioctl cmd 0x%x.\n", cmd); + return -ENOTTY; + } + return 0; +} + +/* maps the PCIe BAR into user space for memory-like access using mmap() */ +int bridge_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct xdma_dev *xdev; + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + unsigned long off; + unsigned long phys; + unsigned long vsize; + unsigned long psize; + int rv; + + rv = xcdev_check(__func__, xcdev, 0); + if (rv < 0) + return rv; + xdev = xcdev->xdev; + + off = vma->vm_pgoff << PAGE_SHIFT; + /* BAR physical address */ + phys = pci_resource_start(xdev->pdev, xcdev->bar) + off; + vsize = vma->vm_end - vma->vm_start; + /* complete resource */ + psize = pci_resource_end(xdev->pdev, xcdev->bar) - + pci_resource_start(xdev->pdev, xcdev->bar) + 1 - off; + + dbg_sg("mmap(): xcdev = 0x%08lx\n", (unsigned long)xcdev); + dbg_sg("mmap(): cdev->bar = %d\n", xcdev->bar); + dbg_sg("mmap(): xdev = 0x%p\n", xdev); + dbg_sg("mmap(): pci_dev = 0x%08lx\n", (unsigned long)xdev->pdev); + + dbg_sg("off = 0x%lx\n", off); + dbg_sg("start = 0x%llx\n", + (unsigned long long)pci_resource_start(xdev->pdev, + xcdev->bar)); + dbg_sg("phys = 0x%lx\n", phys); + + if (vsize > psize) + return -EINVAL; + /* + * pages must not be cached as this would result in cache line sized + * accesses to the end point + */ + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + /* + * prevent touching the pages (byte access) for swap-in, + * and prevent the pages from being swapped out + */ + vma->vm_flags |= VMEM_FLAGS; + /* make MMIO accessible to user space */ + rv = io_remap_pfn_range(vma, vma->vm_start, phys >> PAGE_SHIFT, + vsize, vma->vm_page_prot); + dbg_sg("vma=0x%p, vma->vm_start=0x%lx, phys=0x%lx, size=%lu = %d\n", + vma, vma->vm_start, phys >> PAGE_SHIFT, vsize, rv); + + if (rv) + return -EAGAIN; + return 0; +} + +/* + * character device file operations for control bus (through control bridge) + */ +static const struct file_operations ctrl_fops = { + .owner = THIS_MODULE, + .open = char_open, + .release = char_close, + .read = char_ctrl_read, + .write = char_ctrl_write, + .mmap = bridge_mmap, + .unlocked_ioctl = char_ctrl_ioctl, +}; + +void cdev_ctrl_init(struct xdma_cdev *xcdev) +{ + cdev_init(&xcdev->cdev, &ctrl_fops); +} diff --git a/sources/xdma_driver/xdma/cdev_ctrl.h b/sources/xdma_driver/xdma/cdev_ctrl.h new file mode 100644 index 0000000..cb8ef6d --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_ctrl.h @@ -0,0 +1,80 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef _XDMA_IOCALLS_POSIX_H_ +#define _XDMA_IOCALLS_POSIX_H_ + +#include <linux/ioctl.h> + +/* Use 'x' as magic number */ +#define XDMA_IOC_MAGIC 'x' +/* XL OpenCL X->58(ASCII), L->6C(ASCII), O->0 C->C L->6C(ASCII); */ +#define XDMA_XCL_MAGIC 0X586C0C6C + +/* + * S means "Set" through a ptr, + * T means "Tell" directly with the argument value + * G means "Get": reply by setting through a pointer + * Q means "Query": response is on the return value + * X means "eXchange": switch G and S atomically + * H means "sHift": switch T and Q atomically + * + * _IO(type,nr) no arguments + * _IOR(type,nr,datatype) read data from driver + * _IOW(type,nr.datatype) write data to driver + * _IORW(type,nr,datatype) read/write data + * + * _IOC_DIR(nr) returns direction + * _IOC_TYPE(nr) returns magic + * _IOC_NR(nr) returns number + * _IOC_SIZE(nr) returns size + */ + +enum XDMA_IOC_TYPES { + XDMA_IOC_NOP, + XDMA_IOC_INFO, + XDMA_IOC_OFFLINE, + XDMA_IOC_ONLINE, + XDMA_IOC_MAX +}; + +struct xdma_ioc_base { + unsigned int magic; + unsigned int command; +}; + +struct xdma_ioc_info { + struct xdma_ioc_base base; + unsigned short vendor; + unsigned short device; + unsigned short subsystem_vendor; + unsigned short subsystem_device; + unsigned int dma_engine_version; + unsigned int driver_version; + unsigned long long feature_id; + unsigned short domain; + unsigned char bus; + unsigned char dev; + unsigned char func; +}; + +/* IOCTL codes */ +#define XDMA_IOCINFO _IOWR(XDMA_IOC_MAGIC, XDMA_IOC_INFO, \ + struct xdma_ioc_info) +#define XDMA_IOCOFFLINE _IO(XDMA_IOC_MAGIC, XDMA_IOC_OFFLINE) +#define XDMA_IOCONLINE _IO(XDMA_IOC_MAGIC, XDMA_IOC_ONLINE) + +#define IOCTL_XDMA_ADDRMODE_SET _IOW('q', 4, int) +#define IOCTL_XDMA_ADDRMODE_GET _IOR('q', 5, int) +#define IOCTL_XDMA_ALIGN_GET _IOR('q', 6, int) + +#endif /* _XDMA_IOCALLS_POSIX_H_ */ diff --git a/sources/xdma_driver/xdma/cdev_events.c b/sources/xdma_driver/xdma/cdev_events.c new file mode 100644 index 0000000..3ada7db --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_events.c @@ -0,0 +1,113 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include "xdma_cdev.h" + +/* + * character device file operations for events + */ +static ssize_t char_events_read(struct file *file, char __user *buf, + size_t count, loff_t *pos) +{ + int rv; + struct xdma_user_irq *user_irq; + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + u32 events_user; + unsigned long flags; + + rv = xcdev_check(__func__, xcdev, 0); + if (rv < 0) + return rv; + user_irq = xcdev->user_irq; + if (!user_irq) { + pr_info("xcdev 0x%p, user_irq NULL.\n", xcdev); + return -EINVAL; + } + + if (count != 4) + return -EPROTO; + + if (*pos & 3) + return -EPROTO; + + /* + * sleep until any interrupt events have occurred, + * or a signal arrived + */ + rv = wait_event_interruptible(user_irq->events_wq, + user_irq->events_irq != 0); + if (rv) + dbg_sg("wait_event_interruptible=%d\n", rv); + + /* wait_event_interruptible() was interrupted by a signal */ + if (rv == -ERESTARTSYS) + return -ERESTARTSYS; + + /* atomically decide which events are passed to the user */ + spin_lock_irqsave(&user_irq->events_lock, flags); + events_user = user_irq->events_irq; + user_irq->events_irq = 0; + spin_unlock_irqrestore(&user_irq->events_lock, flags); + + rv = copy_to_user(buf, &events_user, 4); + if (rv) + dbg_sg("Copy to user failed but continuing\n"); + + return 4; +} + +static unsigned int char_events_poll(struct file *file, poll_table *wait) +{ + struct xdma_user_irq *user_irq; + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + unsigned long flags; + unsigned int mask = 0; + int rv; + + rv = xcdev_check(__func__, xcdev, 0); + if (rv < 0) + return rv; + user_irq = xcdev->user_irq; + if (!user_irq) { + pr_info("xcdev 0x%p, user_irq NULL.\n", xcdev); + return -EINVAL; + } + + poll_wait(file, &user_irq->events_wq, wait); + + spin_lock_irqsave(&user_irq->events_lock, flags); + if (user_irq->events_irq) + mask = POLLIN | POLLRDNORM; /* readable */ + + spin_unlock_irqrestore(&user_irq->events_lock, flags); + + return mask; +} + +/* + * character device file operations for the irq events + */ +static const struct file_operations events_fops = { + .owner = THIS_MODULE, + .open = char_open, + .release = char_close, + .read = char_events_read, + .poll = char_events_poll, +}; + +void cdev_event_init(struct xdma_cdev *xcdev) +{ + xcdev->user_irq = &(xcdev->xdev->user_irq[xcdev->bar]); + cdev_init(&xcdev->cdev, &events_fops); +} diff --git a/sources/xdma_driver/xdma/cdev_sgdma.c b/sources/xdma_driver/xdma/cdev_sgdma.c new file mode 100644 index 0000000..410ef45 --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_sgdma.c @@ -0,0 +1,552 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include <asm/cacheflush.h> +#include "libxdma_api.h" +#include "xdma_cdev.h" +#include "cdev_sgdma.h" + +/* Module Parameters */ +unsigned int sgdma_timeout = 10; +module_param(sgdma_timeout, uint, 0644); +MODULE_PARM_DESC(sgdma_timeout, "timeout in seconds for sgdma, default is 10 sec."); + +/* + * character device file operations for SG DMA engine + */ +static loff_t char_sgdma_llseek(struct file *file, loff_t off, int whence) +{ + loff_t newpos = 0; + + switch (whence) { + case 0: /* SEEK_SET */ + newpos = off; + break; + case 1: /* SEEK_CUR */ + newpos = file->f_pos + off; + break; + case 2: /* SEEK_END, @TODO should work from end of address space */ + newpos = UINT_MAX + off; + break; + default: /* can't happen */ + return -EINVAL; + } + if (newpos < 0) + return -EINVAL; + file->f_pos = newpos; + dbg_fops("char_sgdma_llseek: pos=%lld\n", (signed long long)newpos); + +#if 0 + pr_err("0x%p, off 0x%lld, whence %d -> pos %lld.\n", + file, (signed long long)off, whence, (signed long long)off); +#endif + + return newpos; +} + +/* char_sgdma_read_write() -- Read from or write to the device + * + * @buf userspace buffer + * @count number of bytes in the userspace buffer + * @pos byte-address in device + * @dir_to_device If !0, a write to the device is performed + * + * Iterate over the userspace buffer, taking at most 255 * PAGE_SIZE bytes for + * each DMA transfer. + * + * For each transfer, get the user pages, build a sglist, map, build a + * descriptor table. submit the transfer. wait for the interrupt handler + * to wake us on completion. + */ + +static int check_transfer_align(struct xdma_engine *engine, + const char __user *buf, size_t count, loff_t pos, int sync) +{ + BUG_ON(!engine); + + /* AXI ST or AXI MM non-incremental addressing mode? */ + if (engine->non_incr_addr) { + int buf_lsb = (int)((uintptr_t)buf) & (engine->addr_align - 1); + size_t len_lsb = count & ((size_t)engine->len_granularity - 1); + int pos_lsb = (int)pos & (engine->addr_align - 1); + + dbg_tfr("AXI ST or MM non-incremental\n"); + dbg_tfr("buf_lsb = %d, pos_lsb = %d, len_lsb = %ld\n", buf_lsb, + pos_lsb, len_lsb); + + if (buf_lsb != 0) { + dbg_tfr("FAIL: non-aligned buffer address %p\n", buf); + return -EINVAL; + } + + if ((pos_lsb != 0) && (sync)) { + dbg_tfr("FAIL: non-aligned AXI MM FPGA addr 0x%llx\n", + (unsigned long long)pos); + return -EINVAL; + } + + if (len_lsb != 0) { + dbg_tfr("FAIL: len %d is not a multiple of %d\n", + (int)count, + (int)engine->len_granularity); + return -EINVAL; + } + /* AXI MM incremental addressing mode */ + } else { + int buf_lsb = (int)((uintptr_t)buf) & (engine->addr_align - 1); + int pos_lsb = (int)pos & (engine->addr_align - 1); + + if (buf_lsb != pos_lsb) { + dbg_tfr("FAIL: Misalignment error\n"); + dbg_tfr("host addr %p, FPGA addr 0x%llx\n", buf, pos); + return -EINVAL; + } + } + + return 0; +} + +/* + * Map a user memory range into a scatterlist + * inspired by vhost_scsi_map_to_sgl() + * Returns the number of scatterlist entries used or -errno on error. + */ +static inline void xdma_io_cb_release(struct xdma_io_cb *cb) +{ + int i; + + for (i = 0; i < cb->pages_nr; i++) + put_page(cb->pages[i]); + + sg_free_table(&cb->sgt); + kfree(cb->pages); + + memset(cb, 0, sizeof(*cb)); +} + +static void char_sgdma_unmap_user_buf(struct xdma_io_cb *cb, bool write) +{ + int i; + + sg_free_table(&cb->sgt); + + if (!cb->pages || !cb->pages_nr) + return; + + for (i = 0; i < cb->pages_nr; i++) { + if (cb->pages[i]) { + if (!write) + set_page_dirty_lock(cb->pages[i]); + put_page(cb->pages[i]); + } else + break; + } + + if (i != cb->pages_nr) + pr_info("sgl pages %d/%u.\n", i, cb->pages_nr); + + kfree(cb->pages); + cb->pages = NULL; +} + +static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write) +{ + struct sg_table *sgt = &cb->sgt; + unsigned long len = cb->len; + char *buf = cb->buf; + struct scatterlist *sg; + unsigned int pages_nr = (((unsigned long)buf + len + PAGE_SIZE -1) - + ((unsigned long)buf & PAGE_MASK)) + >> PAGE_SHIFT; + int i; + int rv; + + if (pages_nr == 0) { + return -EINVAL; + } + + if (sg_alloc_table(sgt, pages_nr, GFP_KERNEL)) { + pr_err("sgl OOM.\n"); + return -ENOMEM; + } + + cb->pages = kcalloc(pages_nr, sizeof(struct page *), GFP_KERNEL); + if (!cb->pages) { + pr_err("pages OOM.\n"); + rv = -ENOMEM; + goto err_out; + } + + rv = get_user_pages_fast((unsigned long)buf, pages_nr, 1/* write */, + cb->pages); + /* No pages were pinned */ + if (rv < 0) { + pr_err("unable to pin down %u user pages, %d.\n", + pages_nr, rv); + goto err_out; + } + /* Less pages pinned than wanted */ + if (rv != pages_nr) { + pr_err("unable to pin down all %u user pages, %d.\n", + pages_nr, rv); + rv = -EFAULT; + cb->pages_nr = rv; + goto err_out; + } + + for (i = 1; i < pages_nr; i++) { + if (cb->pages[i - 1] == cb->pages[i]) { + pr_err("duplicate pages, %d, %d.\n", + i - 1, i); + rv = -EFAULT; + cb->pages_nr = pages_nr; + goto err_out; + } + } + + sg = sgt->sgl; + for (i = 0; i < pages_nr; i++, sg = sg_next(sg)) { + //unsigned int offset = (uintptr_t)buf & ~PAGE_MASK; + unsigned int offset = offset_in_page(buf); + unsigned int nbytes = min_t(unsigned int, PAGE_SIZE - offset, len); + + flush_dcache_page(cb->pages[i]); + sg_set_page(sg, cb->pages[i], nbytes, offset); + + buf += nbytes; + len -= nbytes; + } + + BUG_ON(len); + cb->pages_nr = pages_nr; + return 0; + +err_out: + char_sgdma_unmap_user_buf(cb, write); + + return rv; +} + +static ssize_t char_sgdma_read_write(struct file *file, char __user *buf, + size_t count, loff_t *pos, bool write) +{ + int rv; + ssize_t res = 0; + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + struct xdma_dev *xdev; + struct xdma_engine *engine; + struct xdma_io_cb cb; + + rv = xcdev_check(__func__, xcdev, 1); + if (rv < 0) + return rv; + xdev = xcdev->xdev; + engine = xcdev->engine; + + dbg_tfr("file 0x%p, priv 0x%p, buf 0x%p,%llu, pos %llu, W %d, %s.\n", + file, file->private_data, buf, (u64)count, (u64)*pos, write, + engine->name); + + if ((write && engine->dir != DMA_TO_DEVICE) || + (!write && engine->dir != DMA_FROM_DEVICE)) { + pr_err("r/w mismatch. W %d, dir %d.\n", + write, engine->dir); + return -EINVAL; + } + + rv = check_transfer_align(engine, buf, count, *pos, 1); + if (rv) { + pr_info("Invalid transfer alignment detected\n"); + return rv; + } + + memset(&cb, 0, sizeof(struct xdma_io_cb)); + cb.buf = buf; + cb.len = count; + rv = char_sgdma_map_user_buf_to_sgl(&cb, write); + if (rv < 0) + return rv; + + res = xdma_xfer_submit(xdev, engine->channel, write, *pos, &cb.sgt, + 0, sgdma_timeout * 1000); + //pr_err("xfer_submit return=%lld.\n", (s64)res); + + //interrupt_status(xdev); + + char_sgdma_unmap_user_buf(&cb, write); + + return res; +} + + +static ssize_t char_sgdma_write(struct file *file, const char __user *buf, + size_t count, loff_t *pos) +{ + return char_sgdma_read_write(file, (char *)buf, count, pos, 1); +} + +static ssize_t char_sgdma_read(struct file *file, char __user *buf, + size_t count, loff_t *pos) +{ + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + struct xdma_engine *engine; + int rv; + + rv = xcdev_check(__func__, xcdev, 1); + if (rv < 0) + return rv; + + engine = xcdev->engine; + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { + rv = xdma_cyclic_transfer_setup(engine); + if (rv < 0 && rv != -EBUSY) + return rv; + /* 600 sec. timeout */ + return xdma_engine_read_cyclic(engine, buf, count, 600000); + } + + return char_sgdma_read_write(file, (char *)buf, count, pos, 0); +} + +static int ioctl_do_perf_start(struct xdma_engine *engine, unsigned long arg) +{ + int rv; + struct xdma_dev *xdev; + + BUG_ON(!engine); + xdev = engine->xdev; + BUG_ON(!xdev); + + /* performance measurement already running on this engine? */ + if (engine->xdma_perf) { + dbg_perf("IOCTL_XDMA_PERF_START failed!\n"); + dbg_perf("Perf measurement already seems to be running!\n"); + return -EBUSY; + } + engine->xdma_perf = kzalloc(sizeof(struct xdma_performance_ioctl), + GFP_KERNEL); + + if (!engine->xdma_perf) + return -ENOMEM; + + rv = copy_from_user(engine->xdma_perf, + (struct xdma_performance_ioctl *)arg, + sizeof(struct xdma_performance_ioctl)); + + if (rv < 0) { + dbg_perf("Failed to copy from user space 0x%lx\n", arg); + return -EINVAL; + } + if (engine->xdma_perf->version != IOCTL_XDMA_PERF_V1) { + dbg_perf("Unsupported IOCTL version %d\n", + engine->xdma_perf->version); + return -EINVAL; + } + + enable_perf(engine); + dbg_perf("transfer_size = %d\n", engine->xdma_perf->transfer_size); + /* initialize wait queue */ + init_waitqueue_head(&engine->xdma_perf_wq); + xdma_performance_submit(xdev, engine); + + return 0; +} + +static int ioctl_do_perf_stop(struct xdma_engine *engine, unsigned long arg) +{ + struct xdma_transfer *transfer = NULL; + int rv; + + dbg_perf("IOCTL_XDMA_PERF_STOP\n"); + + /* no performance measurement running on this engine? */ + if (!engine->xdma_perf) { + dbg_perf("No measurement in progress\n"); + return -EINVAL; + } + + /* stop measurement */ + transfer = engine_cyclic_stop(engine); + dbg_perf("Waiting for measurement to stop\n"); + + if (engine->xdma_perf) { + get_perf_stats(engine); + + rv = copy_to_user((void __user *)arg, engine->xdma_perf, + sizeof(struct xdma_performance_ioctl)); + if (rv) { + dbg_perf("Error copying result to user\n"); + return -EINVAL; + } + } else { + dbg_perf("engine->xdma_perf == NULL?\n"); + } + + kfree(engine->xdma_perf); + engine->xdma_perf = NULL; + + return 0; +} + +static int ioctl_do_perf_get(struct xdma_engine *engine, unsigned long arg) +{ + int rc; + + BUG_ON(!engine); + + dbg_perf("IOCTL_XDMA_PERF_GET\n"); + + if (engine->xdma_perf) { + get_perf_stats(engine); + + rc = copy_to_user((void __user *)arg, engine->xdma_perf, + sizeof(struct xdma_performance_ioctl)); + if (rc) { + dbg_perf("Error copying result to user\n"); + return -EINVAL; + } + } else { + dbg_perf("engine->xdma_perf == NULL?\n"); + return -EPROTO; + } + + return 0; +} + +static int ioctl_do_addrmode_set(struct xdma_engine *engine, unsigned long arg) +{ + return engine_addrmode_set(engine, arg); +} + +static int ioctl_do_addrmode_get(struct xdma_engine *engine, unsigned long arg) +{ + int rv; + unsigned long src; + + BUG_ON(!engine); + src = !!engine->non_incr_addr; + + dbg_perf("IOCTL_XDMA_ADDRMODE_GET\n"); + rv = put_user(src, (int __user *)arg); + + return rv; +} + +static int ioctl_do_align_get(struct xdma_engine *engine, unsigned long arg) +{ + BUG_ON(!engine); + + dbg_perf("IOCTL_XDMA_ALIGN_GET\n"); + return put_user(engine->addr_align, (int __user *)arg); +} + +static long char_sgdma_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + struct xdma_dev *xdev; + struct xdma_engine *engine; + + int rv = 0; + + rv = xcdev_check(__func__, xcdev, 1); + if (rv < 0) + return rv; + + xdev = xcdev->xdev; + engine = xcdev->engine; + + switch (cmd) { + case IOCTL_XDMA_PERF_START: + rv = ioctl_do_perf_start(engine, arg); + break; + case IOCTL_XDMA_PERF_STOP: + rv = ioctl_do_perf_stop(engine, arg); + break; + case IOCTL_XDMA_PERF_GET: + rv = ioctl_do_perf_get(engine, arg); + break; + case IOCTL_XDMA_ADDRMODE_SET: + rv = ioctl_do_addrmode_set(engine, arg); + break; + case IOCTL_XDMA_ADDRMODE_GET: + rv = ioctl_do_addrmode_get(engine, arg); + break; + case IOCTL_XDMA_ALIGN_GET: + rv = ioctl_do_align_get(engine, arg); + break; + default: + dbg_perf("Unsupported operation\n"); + rv = -EINVAL; + break; + } + + return rv; +} + +static int char_sgdma_open(struct inode *inode, struct file *file) +{ + struct xdma_cdev *xcdev; + struct xdma_engine *engine; + + char_open(inode, file); + + xcdev = (struct xdma_cdev *)file->private_data; + engine = xcdev->engine; + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { + if (engine->device_open == 1) + return -EBUSY; + else + engine->device_open = 1; + } + + return 0; +} + +static int char_sgdma_close(struct inode *inode, struct file *file) +{ + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + struct xdma_engine *engine; + int rv; + + rv = xcdev_check(__func__, xcdev, 1); + if (rv < 0) + return rv; + + engine = xcdev->engine; + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { + engine->device_open = 0; + if (engine->cyclic_req) + return xdma_cyclic_transfer_teardown(engine); + } + + return 0; +} +static const struct file_operations sgdma_fops = { + .owner = THIS_MODULE, + .open = char_sgdma_open, + .release = char_sgdma_close, + .write = char_sgdma_write, + .read = char_sgdma_read, + .unlocked_ioctl = char_sgdma_ioctl, + .llseek = char_sgdma_llseek, +}; + +void cdev_sgdma_init(struct xdma_cdev *xcdev) +{ + cdev_init(&xcdev->cdev, &sgdma_fops); +} diff --git a/sources/xdma_driver/xdma/cdev_sgdma.h b/sources/xdma_driver/xdma/cdev_sgdma.h new file mode 100644 index 0000000..7781fae --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_sgdma.h @@ -0,0 +1,66 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef _XDMA_IOCALLS_POSIX_H_ +#define _XDMA_IOCALLS_POSIX_H_ + +#include <linux/ioctl.h> + + +#define IOCTL_XDMA_PERF_V1 (1) +#define XDMA_ADDRMODE_MEMORY (0) +#define XDMA_ADDRMODE_FIXED (1) + +/* + * S means "Set" through a ptr, + * T means "Tell" directly with the argument value + * G means "Get": reply by setting through a pointer + * Q means "Query": response is on the return value + * X means "eXchange": switch G and S atomically + * H means "sHift": switch T and Q atomically + * + * _IO(type,nr) no arguments + * _IOR(type,nr,datatype) read data from driver + * _IOW(type,nr.datatype) write data to driver + * _IORW(type,nr,datatype) read/write data + * + * _IOC_DIR(nr) returns direction + * _IOC_TYPE(nr) returns magic + * _IOC_NR(nr) returns number + * _IOC_SIZE(nr) returns size + */ + +struct xdma_performance_ioctl +{ + /* IOCTL_XDMA_IOCTL_Vx */ + uint32_t version; + uint32_t transfer_size; + /* measurement */ + uint32_t stopped; + uint32_t iterations; + uint64_t clock_cycle_count; + uint64_t data_cycle_count; + uint64_t pending_count; +}; + + + +/* IOCTL codes */ + +#define IOCTL_XDMA_PERF_START _IOW('q', 1, struct xdma_performance_ioctl *) +#define IOCTL_XDMA_PERF_STOP _IOW('q', 2, struct xdma_performance_ioctl *) +#define IOCTL_XDMA_PERF_GET _IOR('q', 3, struct xdma_performance_ioctl *) +#define IOCTL_XDMA_ADDRMODE_SET _IOW('q', 4, int) +#define IOCTL_XDMA_ADDRMODE_GET _IOR('q', 5, int) +#define IOCTL_XDMA_ALIGN_GET _IOR('q', 6, int) + +#endif /* _XDMA_IOCALLS_POSIX_H_ */ diff --git a/sources/xdma_driver/xdma/cdev_xvc.c b/sources/xdma_driver/xdma/cdev_xvc.c new file mode 100644 index 0000000..a077bb1 --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_xvc.c @@ -0,0 +1,238 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include "xdma_cdev.h" +#include "cdev_xvc.h" + +#define COMPLETION_LOOP_MAX 100 + +#define XVC_BAR_LENGTH_REG 0x0 +#define XVC_BAR_TMS_REG 0x4 +#define XVC_BAR_TDI_REG 0x8 +#define XVC_BAR_TDO_REG 0xC +#define XVC_BAR_CTRL_REG 0x10 + +#ifdef __REG_DEBUG__ +/* SECTION: Function definitions */ +inline void __write_register(const char *fn, u32 value, void *base, + unsigned int off) +{ + pr_info("%s: 0x%p, W reg 0x%lx, 0x%x.\n", fn, base, off, value); + iowrite32(value, base + off); +} + +inline u32 __read_register(const char *fn, void *base, unsigned int off) +{ + u32 v = ioread32(base + off); + + pr_info("%s: 0x%p, R reg 0x%lx, 0x%x.\n", fn, base, off, v); + return v; +} +#define write_register(v,base,off) __write_register(__func__, v, base, off) +#define read_register(base,off) __read_register(__func__, base, off) + +#else +#define write_register(v,base,off) iowrite32(v, (base) + (off)) +#define read_register(base,off) ioread32((base) + (off)) +#endif /* #ifdef __REG_DEBUG__ */ + + +static int xvc_shift_bits(void *base, u32 tms_bits, u32 tdi_bits, + u32 *tdo_bits) +{ + u32 control; + int count; + + /* set tms bit */ + write_register(tms_bits, base, XVC_BAR_TMS_REG); + /* set tdi bits and shift data out */ + write_register(tdi_bits, base, XVC_BAR_TDI_REG); + /* enable shift operation */ + write_register(0x1, base, XVC_BAR_CTRL_REG); + + /* poll for completion */ + count = COMPLETION_LOOP_MAX; + while (count) { + /* read control reg to check shift operation completion */ + control = read_register(base, XVC_BAR_CTRL_REG); + if ((control & 0x01) == 0) + break; + + count--; + } + + if (!count) { + pr_warn("XVC bar transaction timed out (0x%0X)\n", control); + return -ETIMEDOUT; + } + + /* read tdo bits back out */ + *tdo_bits = read_register(base, XVC_BAR_TDO_REG); + + return 0; +} + +static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct xdma_cdev *xcdev = (struct xdma_cdev *)filp->private_data; + struct xdma_dev *xdev; + struct xvc_ioc xvc_obj; + unsigned int opcode; + unsigned int total_bits; + unsigned int total_bytes; + unsigned char *buffer = NULL; + unsigned char *tms_buf = NULL; + unsigned char *tdi_buf = NULL; + unsigned char *tdo_buf = NULL; + unsigned int bits, bits_left; + void __iomem *iobase; + int rv; + + rv = xcdev_check(__func__, xcdev, 0); + if (rv < 0) + return rv; + xdev = xcdev->xdev; + + if (cmd != XDMA_IOCXVC) { + pr_info("ioctl 0x%x, UNKNOWN cmd.\n", cmd); + return -ENOIOCTLCMD; + } + + rv = copy_from_user((void *)&xvc_obj, (void __user *)arg, + sizeof(struct xvc_ioc)); + /* anything not copied ? */ + if (rv) { + pr_info("copy_from_user xvc_obj failed: %d.\n", rv); + goto cleanup; + } + + opcode = xvc_obj.opcode; + + /* Invalid operation type, no operation performed */ + if (opcode != 0x01 && opcode != 0x02) { + pr_info("UNKNOWN opcode 0x%x.\n", opcode); + return -EINVAL; + } + + total_bits = xvc_obj.length; + total_bytes = (total_bits + 7) >> 3; + + buffer = (char *)kmalloc(total_bytes * 3, GFP_KERNEL); + if (!buffer) { + pr_info("OOM %u, op 0x%x, len %u bits, %u bytes.\n", + 3 * total_bytes, opcode, total_bits, total_bytes); + rv = -ENOMEM; + goto cleanup; + } + tms_buf = buffer; + tdi_buf = tms_buf + total_bytes; + tdo_buf = tdi_buf + total_bytes; + + rv = copy_from_user((void *)tms_buf, xvc_obj.tms_buf, total_bytes); + if (rv) { + pr_info("copy tmfs_buf failed: %d/%u.\n", rv, total_bytes); + goto cleanup; + } + rv = copy_from_user((void *)tdi_buf, xvc_obj.tdi_buf, total_bytes); + if (rv) { + pr_info("copy tdi_buf failed: %d/%u.\n", rv, total_bytes); + goto cleanup; + } + + /* exclusive access */ + spin_lock(&xcdev->lock); + + iobase = xdev->bar[xcdev->bar] + xcdev->base; + /* set length register to 32 initially if more than one + * word-transaction is to be done */ + if (total_bits >= 32) + write_register(0x20, iobase, XVC_BAR_LENGTH_REG); + + for (bits = 0, bits_left = total_bits; bits < total_bits; bits += 32, + bits_left -= 32) { + unsigned int bytes = bits >> 3; + unsigned int shift_bytes = 4; + u32 tms_store = 0; + u32 tdi_store = 0; + u32 tdo_store = 0; + + if (bits_left < 32) { + /* set number of bits to shift out */ + write_register(bits_left, iobase, XVC_BAR_LENGTH_REG); + shift_bytes = (bits_left + 7) >> 3; + } + + memcpy(&tms_store, tms_buf + bytes, shift_bytes); + memcpy(&tdi_store, tdi_buf + bytes, shift_bytes); + + /* Shift data out and copy to output buffer */ + rv = xvc_shift_bits(iobase, tms_store, tdi_store, &tdo_store); + if (rv < 0) + goto cleanup; + + memcpy(tdo_buf + bytes, &tdo_store, shift_bytes); + } + + /* if testing bar access swap tdi and tdo bufferes to "loopback" */ + if (opcode == 0x2) { + char *tmp = tdo_buf; + + tdo_buf = tdi_buf; + tdi_buf = tmp; + } + + rv = copy_to_user((void *)xvc_obj.tdo_buf, tdo_buf, total_bytes); + if (rv) { + pr_info("copy back tdo_buf failed: %d/%u.\n", rv, total_bytes); + rv = -EFAULT; + goto cleanup; + } + +cleanup: + if (buffer) + kfree(buffer); + + #if LINUX_VERSION_CODE < KERNEL_VERSION(5,3,0) + mmiowb(); + #endif + + spin_unlock(&xcdev->lock); + + return rv; +} + +/* + * character device file operations for the XVC + */ +static const struct file_operations xvc_fops = { + .owner = THIS_MODULE, + .open = char_open, + .release = char_close, + .unlocked_ioctl = xvc_ioctl, +}; + +void cdev_xvc_init(struct xdma_cdev *xcdev) +{ +#ifdef __XVC_BAR_NUM__ + xcdev->bar = __XVC_BAR_NUM__; +#endif +#ifdef __XVC_BAR_OFFSET__ + xcdev->base = __XVC_BAR_OFFSET__; +#else + xcdev->base = XVC_BAR_OFFSET_DFLT; +#endif + pr_info("xcdev 0x%p, bar %u, offset 0x%lx.\n", + xcdev, xcdev->bar, xcdev->base); + cdev_init(&xcdev->cdev, &xvc_fops); +} diff --git a/sources/xdma_driver/xdma/cdev_xvc.h b/sources/xdma_driver/xdma/cdev_xvc.h new file mode 100644 index 0000000..e689706 --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_xvc.h @@ -0,0 +1,36 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef __XVC_IOCTL_H__ +#define __XVC_IOCTL_H__ + +#include <linux/ioctl.h> + +/* + * !!! TODO !!! + * need a better way set the bar offset dynamicly + */ +#define XVC_BAR_OFFSET_DFLT 0x40000 /* DSA 4.0 */ + +#define XVC_MAGIC 0x58564344 // "XVCD" + +struct xvc_ioc { + unsigned int opcode; + unsigned int length; + unsigned char *tms_buf; + unsigned char *tdi_buf; + unsigned char *tdo_buf; +}; + +#define XDMA_IOCXVC _IOWR(XVC_MAGIC, 1, struct xvc_ioc) + +#endif /* __XVC_IOCTL_H__ */ diff --git a/sources/xdma_driver/xdma/cdev_xvc.o.ur-safe b/sources/xdma_driver/xdma/cdev_xvc.o.ur-safe new file mode 100644 index 0000000..3a9a009 --- /dev/null +++ b/sources/xdma_driver/xdma/cdev_xvc.o.ur-safe @@ -0,0 +1 @@ +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/cdev_xvc.o-.text-315 diff --git a/sources/xdma_driver/xdma/libxdma.c b/sources/xdma_driver/xdma/libxdma.c new file mode 100644 index 0000000..1743780 --- /dev/null +++ b/sources/xdma_driver/xdma/libxdma.c @@ -0,0 +1,4445 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/string.h> +#include <linux/mm.h> +#include <linux/errno.h> +#include <linux/sched.h> +#include <linux/vmalloc.h> + +#include "libxdma.h" +#include "libxdma_api.h" +#include "cdev_sgdma.h" + +/* SECTION: Module licensing */ + +#ifdef __LIBXDMA_MOD__ +#include "version.h" +#define DRV_MODULE_NAME "libxdma" +#define DRV_MODULE_DESC "Xilinx XDMA Base Driver" +#define DRV_MODULE_RELDATE "Feb. 2018" + +static char version[] = + DRV_MODULE_DESC " " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n"; + +MODULE_AUTHOR("Xilinx, Inc."); +MODULE_DESCRIPTION(DRV_MODULE_DESC); +MODULE_VERSION(DRV_MODULE_VERSION); +MODULE_LICENSE("Dual BSD/GPL"); +#endif + +/* Module Parameters */ +static unsigned int poll_mode; +module_param(poll_mode, uint, 0644); +MODULE_PARM_DESC(poll_mode, "Set 1 for hw polling, default is 0 (interrupts)"); + +static unsigned int interrupt_mode; +module_param(interrupt_mode, uint, 0644); +MODULE_PARM_DESC(interrupt_mode, "0 - MSI-x , 1 - MSI, 2 - Legacy"); + +static unsigned int enable_credit_mp; +module_param(enable_credit_mp, uint, 0644); +MODULE_PARM_DESC(enable_credit_mp, "Set 1 to enable creidt feature, default is 0 (no credit control)"); + +unsigned int desc_blen_max = XDMA_DESC_BLEN_MAX; +module_param(desc_blen_max, uint, 0644); +MODULE_PARM_DESC(desc_blen_max, "per descriptor max. buffer length, default is (1 << 28) - 1"); + +/* + * xdma device management + * maintains a list of the xdma devices + */ +static LIST_HEAD(xdev_list); +static DEFINE_MUTEX(xdev_mutex); + +static LIST_HEAD(xdev_rcu_list); +static DEFINE_SPINLOCK(xdev_rcu_lock); + +#ifndef list_last_entry +#define list_last_entry(ptr, type, member) \ + list_entry((ptr)->prev, type, member) +#endif + +static inline void xdev_list_add(struct xdma_dev *xdev) +{ + mutex_lock(&xdev_mutex); + if (list_empty(&xdev_list)) + xdev->idx = 0; + else { + struct xdma_dev *last; + + last = list_last_entry(&xdev_list, struct xdma_dev, list_head); + xdev->idx = last->idx + 1; + } + list_add_tail(&xdev->list_head, &xdev_list); + mutex_unlock(&xdev_mutex); + + dbg_init("dev %s, xdev 0x%p, xdma idx %d.\n", + dev_name(&xdev->pdev->dev), xdev, xdev->idx); + + spin_lock(&xdev_rcu_lock); + list_add_tail_rcu(&xdev->rcu_node, &xdev_rcu_list); + spin_unlock(&xdev_rcu_lock); +} + +#undef list_last_entry + +static inline void xdev_list_remove(struct xdma_dev *xdev) +{ + mutex_lock(&xdev_mutex); + list_del(&xdev->list_head); + mutex_unlock(&xdev_mutex); + + spin_lock(&xdev_rcu_lock); + list_del_rcu(&xdev->rcu_node); + spin_unlock(&xdev_rcu_lock); + synchronize_rcu(); +} + +struct xdma_dev *xdev_find_by_pdev(struct pci_dev *pdev) +{ + struct xdma_dev *xdev, *tmp; + + mutex_lock(&xdev_mutex); + list_for_each_entry_safe(xdev, tmp, &xdev_list, list_head) { + if (xdev->pdev == pdev) { + mutex_unlock(&xdev_mutex); + return xdev; + } + } + mutex_unlock(&xdev_mutex); + return NULL; +} +EXPORT_SYMBOL_GPL(xdev_find_by_pdev); + +static inline int debug_check_dev_hndl(const char *fname, struct pci_dev *pdev, + void *hndl) +{ + struct xdma_dev *xdev; + + if (!pdev) + return -EINVAL; + + xdev = xdev_find_by_pdev(pdev); + if (!xdev) { + pr_info("%s pdev 0x%p, hndl 0x%p, NO match found!\n", + fname, pdev, hndl); + return -EINVAL; + } + if (xdev != hndl) { + pr_err("%s pdev 0x%p, hndl 0x%p != 0x%p!\n", + fname, pdev, hndl, xdev); + return -EINVAL; + } + + return 0; +} + +#ifdef __LIBXDMA_DEBUG__ +/* SECTION: Function definitions */ +inline void __write_register(const char *fn, u32 value, void *iomem, unsigned long off) +{ + pr_err("%s: w reg 0x%lx(0x%p), 0x%x.\n", fn, off, iomem, value); + iowrite32(value, iomem); +} +#define write_register(v,mem,off) __write_register(__func__, v, mem, off) +#else +#define write_register(v,mem,off) iowrite32(v, mem) +#endif + +inline u32 read_register(void *iomem) +{ + return ioread32(iomem); +} + +static inline u32 build_u32(u32 hi, u32 lo) +{ + return ((hi & 0xFFFFUL) << 16) | (lo & 0xFFFFUL); +} + +static inline u64 build_u64(u64 hi, u64 lo) +{ + return ((hi & 0xFFFFFFFULL) << 32) | (lo & 0xFFFFFFFFULL); +} + +static void check_nonzero_interrupt_status(struct xdma_dev *xdev) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + u32 w; + + w = read_register(®->user_int_enable); + if (w) + pr_info("%s xdma%d user_int_enable = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + + w = read_register(®->channel_int_enable); + if (w) + pr_info("%s xdma%d channel_int_enable = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + + w = read_register(®->user_int_request); + if (w) + pr_info("%s xdma%d user_int_request = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + w = read_register(®->channel_int_request); + if (w) + pr_info("%s xdma%d channel_int_request = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + + w = read_register(®->user_int_pending); + if (w) + pr_info("%s xdma%d user_int_pending = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); + w = read_register(®->channel_int_pending); + if (w) + pr_info("%s xdma%d channel_int_pending = 0x%08x\n", + dev_name(&xdev->pdev->dev), xdev->idx, w); +} + +/* channel_interrupts_enable -- Enable interrupts we are interested in */ +static void channel_interrupts_enable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->channel_int_enable_w1s, XDMA_OFS_INT_CTRL); +} + +/* channel_interrupts_disable -- Disable interrupts we not interested in */ +static void channel_interrupts_disable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->channel_int_enable_w1c, XDMA_OFS_INT_CTRL); +} + +/* user_interrupts_enable -- Enable interrupts we are interested in */ +static void user_interrupts_enable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->user_int_enable_w1s, XDMA_OFS_INT_CTRL); +} + +/* user_interrupts_disable -- Disable interrupts we not interested in */ +static void user_interrupts_disable(struct xdma_dev *xdev, u32 mask) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + + write_register(mask, ®->user_int_enable_w1c, XDMA_OFS_INT_CTRL); +} + +/* read_interrupts -- Print the interrupt controller status */ +static u32 read_interrupts(struct xdma_dev *xdev) +{ + struct interrupt_regs *reg = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); + u32 lo; + u32 hi; + + /* extra debugging; inspect complete engine set of registers */ + hi = read_register(®->user_int_request); + dbg_io("ioread32(0x%p) returned 0x%08x (user_int_request).\n", + ®->user_int_request, hi); + lo = read_register(®->channel_int_request); + dbg_io("ioread32(0x%p) returned 0x%08x (channel_int_request)\n", + ®->channel_int_request, lo); + + /* return interrupts: user in upper 16-bits, channel in lower 16-bits */ + return build_u32(hi, lo); +} + +void enable_perf(struct xdma_engine *engine) +{ + u32 w; + + w = XDMA_PERF_CLEAR; + write_register(w, &engine->regs->perf_ctrl, + (unsigned long)(&engine->regs->perf_ctrl) - + (unsigned long)(&engine->regs)); + read_register(&engine->regs->identifier); + w = XDMA_PERF_AUTO | XDMA_PERF_RUN; + write_register(w, &engine->regs->perf_ctrl, + (unsigned long)(&engine->regs->perf_ctrl) - + (unsigned long)(&engine->regs)); + read_register(&engine->regs->identifier); + + dbg_perf("IOCTL_XDMA_PERF_START\n"); + +} +EXPORT_SYMBOL_GPL(enable_perf); + +void get_perf_stats(struct xdma_engine *engine) +{ + u32 hi; + u32 lo; + + BUG_ON(!engine); + + if (!engine->xdma_perf) { + pr_info("%s perf struct not set up.\n", engine->name); + return; + } + + hi = 0; + lo = read_register(&engine->regs->completed_desc_count); + engine->xdma_perf->iterations = build_u64(hi, lo); + + hi = read_register(&engine->regs->perf_cyc_hi); + lo = read_register(&engine->regs->perf_cyc_lo); + + engine->xdma_perf->clock_cycle_count = build_u64(hi, lo); + + hi = read_register(&engine->regs->perf_dat_hi); + lo = read_register(&engine->regs->perf_dat_lo); + engine->xdma_perf->data_cycle_count = build_u64(hi, lo); + + hi = read_register(&engine->regs->perf_pnd_hi); + lo = read_register(&engine->regs->perf_pnd_lo); + engine->xdma_perf->pending_count = build_u64(hi, lo); +} +EXPORT_SYMBOL_GPL(get_perf_stats); + +static void engine_reg_dump(struct xdma_engine *engine) +{ + u32 w; + + BUG_ON(!engine); + + w = read_register(&engine->regs->identifier); + pr_info("%s: ioread32(0x%p) = 0x%08x (id).\n", + engine->name, &engine->regs->identifier, w); + w &= BLOCK_ID_MASK; + if (w != BLOCK_ID_HEAD) { + pr_info("%s: engine id missing, 0x%08x exp. & 0x%x = 0x%x\n", + engine->name, w, BLOCK_ID_MASK, BLOCK_ID_HEAD); + return; + } + /* extra debugging; inspect complete engine set of registers */ + w = read_register(&engine->regs->status); + pr_info("%s: ioread32(0x%p) = 0x%08x (status).\n", + engine->name, &engine->regs->status, w); + w = read_register(&engine->regs->control); + pr_info("%s: ioread32(0x%p) = 0x%08x (control)\n", + engine->name, &engine->regs->control, w); + w = read_register(&engine->sgdma_regs->first_desc_lo); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_lo)\n", + engine->name, &engine->sgdma_regs->first_desc_lo, w); + w = read_register(&engine->sgdma_regs->first_desc_hi); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_hi)\n", + engine->name, &engine->sgdma_regs->first_desc_hi, w); + w = read_register(&engine->sgdma_regs->first_desc_adjacent); + pr_info("%s: ioread32(0x%p) = 0x%08x (first_desc_adjacent).\n", + engine->name, &engine->sgdma_regs->first_desc_adjacent, w); + w = read_register(&engine->regs->completed_desc_count); + pr_info("%s: ioread32(0x%p) = 0x%08x (completed_desc_count).\n", + engine->name, &engine->regs->completed_desc_count, w); + w = read_register(&engine->regs->interrupt_enable_mask); + pr_info("%s: ioread32(0x%p) = 0x%08x (interrupt_enable_mask)\n", + engine->name, &engine->regs->interrupt_enable_mask, w); +} + +/** + * engine_status_read() - read status of SG DMA engine (optionally reset) + * + * Stores status in engine->status. + * + * @return -1 on failure, status register otherwise + */ +static void engine_status_dump(struct xdma_engine *engine) +{ + u32 v = engine->status; + char buffer[256]; + char *buf = buffer; + int len = 0; + + len = sprintf(buf, "SG engine %s status: 0x%08x: ", engine->name, v); + + if ((v & XDMA_STAT_BUSY)) + len += sprintf(buf + len, "BUSY,"); + if ((v & XDMA_STAT_DESC_STOPPED)) + len += sprintf(buf + len, "DESC_STOPPED,"); + if ((v & XDMA_STAT_DESC_COMPLETED)) + len += sprintf(buf + len, "DESC_COMPL,"); + + /* common H2C & C2H */ + if ((v & XDMA_STAT_COMMON_ERR_MASK)) { + if ((v & XDMA_STAT_ALIGN_MISMATCH)) + len += sprintf(buf + len, "ALIGN_MISMATCH "); + if ((v & XDMA_STAT_MAGIC_STOPPED)) + len += sprintf(buf + len, "MAGIC_STOPPED "); + if ((v & XDMA_STAT_INVALID_LEN)) + len += sprintf(buf + len, "INVLIAD_LEN "); + if ((v & XDMA_STAT_IDLE_STOPPED)) + len += sprintf(buf + len, "IDLE_STOPPED "); + buf[len - 1] = ','; + } + + if ((engine->dir == DMA_TO_DEVICE)) { + /* H2C only */ + if ((v & XDMA_STAT_H2C_R_ERR_MASK)) { + len += sprintf(buf + len, "R:"); + if ((v & XDMA_STAT_H2C_R_UNSUPP_REQ)) + len += sprintf(buf + len, "UNSUPP_REQ "); + if ((v & XDMA_STAT_H2C_R_COMPL_ABORT)) + len += sprintf(buf + len, "COMPL_ABORT "); + if ((v & XDMA_STAT_H2C_R_PARITY_ERR)) + len += sprintf(buf + len, "PARITY "); + if ((v & XDMA_STAT_H2C_R_HEADER_EP)) + len += sprintf(buf + len, "HEADER_EP "); + if ((v & XDMA_STAT_H2C_R_UNEXP_COMPL)) + len += sprintf(buf + len, "UNEXP_COMPL "); + buf[len - 1] = ','; + } + + if ((v & XDMA_STAT_H2C_W_ERR_MASK)) { + len += sprintf(buf + len, "W:"); + if ((v & XDMA_STAT_H2C_W_DECODE_ERR)) + len += sprintf(buf + len, "DECODE_ERR "); + if ((v & XDMA_STAT_H2C_W_SLAVE_ERR)) + len += sprintf(buf + len, "SLAVE_ERR "); + buf[len - 1] = ','; + } + + } else { + /* C2H only */ + if ((v & XDMA_STAT_C2H_R_ERR_MASK)) { + len += sprintf(buf + len, "R:"); + if ((v & XDMA_STAT_C2H_R_DECODE_ERR)) + len += sprintf(buf + len, "DECODE_ERR "); + if ((v & XDMA_STAT_C2H_R_SLAVE_ERR)) + len += sprintf(buf + len, "SLAVE_ERR "); + buf[len - 1] = ','; + } + } + + /* common H2C & C2H */ + if ((v & XDMA_STAT_DESC_ERR_MASK)) { + len += sprintf(buf + len, "DESC_ERR:"); + if ((v & XDMA_STAT_DESC_UNSUPP_REQ)) + len += sprintf(buf + len, "UNSUPP_REQ "); + if ((v & XDMA_STAT_DESC_COMPL_ABORT)) + len += sprintf(buf + len, "COMPL_ABORT "); + if ((v & XDMA_STAT_DESC_PARITY_ERR)) + len += sprintf(buf + len, "PARITY "); + if ((v & XDMA_STAT_DESC_HEADER_EP)) + len += sprintf(buf + len, "HEADER_EP "); + if ((v & XDMA_STAT_DESC_UNEXP_COMPL)) + len += sprintf(buf + len, "UNEXP_COMPL "); + buf[len - 1] = ','; + } + + buf[len - 1] = '\0'; + pr_info("%s\n", buffer); +} + +static u32 engine_status_read(struct xdma_engine *engine, bool clear, bool dump) +{ + u32 value; + + BUG_ON(!engine); + + if (dump) + engine_reg_dump(engine); + + /* read status register */ + if (clear) + value = engine->status = + read_register(&engine->regs->status_rc); + else + value = engine->status = read_register(&engine->regs->status); + + if (dump) + engine_status_dump(engine); + + return value; +} + +/** + * xdma_engine_stop() - stop an SG DMA engine + * + */ +static void xdma_engine_stop(struct xdma_engine *engine) +{ + u32 w; + + BUG_ON(!engine); + dbg_tfr("xdma_engine_stop(engine=%p)\n", engine); + + w = 0; + w |= (u32)XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + w |= (u32)XDMA_CTRL_IE_MAGIC_STOPPED; + w |= (u32)XDMA_CTRL_IE_READ_ERROR; + w |= (u32)XDMA_CTRL_IE_DESC_ERROR; + + if (poll_mode) { + w |= (u32) XDMA_CTRL_POLL_MODE_WB; + } else { + w |= (u32)XDMA_CTRL_IE_DESC_STOPPED; + w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED; + + /* Disable IDLE STOPPED for MM */ + if ((engine->streaming && (engine->dir == DMA_FROM_DEVICE)) || + (engine->xdma_perf)) + w |= (u32)XDMA_CTRL_IE_IDLE_STOPPED; + } + + dbg_tfr("Stopping SG DMA %s engine; writing 0x%08x to 0x%p.\n", + engine->name, w, (u32 *)&engine->regs->control); + write_register(w, &engine->regs->control, + (unsigned long)(&engine->regs->control) - + (unsigned long)(&engine->regs)); + /* dummy read of status register to flush all previous writes */ + dbg_tfr("xdma_engine_stop(%s) done\n", engine->name); +} + +static void engine_start_mode_config(struct xdma_engine *engine) +{ + u32 w; + + BUG_ON(!engine); + + /* If a perf test is running, enable the engine interrupts */ + if (engine->xdma_perf) { + w = XDMA_CTRL_IE_DESC_STOPPED; + w |= XDMA_CTRL_IE_DESC_COMPLETED; + w |= XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + w |= XDMA_CTRL_IE_MAGIC_STOPPED; + w |= XDMA_CTRL_IE_IDLE_STOPPED; + w |= XDMA_CTRL_IE_READ_ERROR; + w |= XDMA_CTRL_IE_DESC_ERROR; + + write_register(w, &engine->regs->interrupt_enable_mask, + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); + } + + /* write control register of SG DMA engine */ + w = (u32)XDMA_CTRL_RUN_STOP; + w |= (u32)XDMA_CTRL_IE_READ_ERROR; + w |= (u32)XDMA_CTRL_IE_DESC_ERROR; + w |= (u32)XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + w |= (u32)XDMA_CTRL_IE_MAGIC_STOPPED; + + if (poll_mode) { + w |= (u32)XDMA_CTRL_POLL_MODE_WB; + } else { + w |= (u32)XDMA_CTRL_IE_DESC_STOPPED; + w |= (u32)XDMA_CTRL_IE_DESC_COMPLETED; + + if ((engine->streaming && (engine->dir == DMA_FROM_DEVICE)) || + (engine->xdma_perf)) + w |= (u32)XDMA_CTRL_IE_IDLE_STOPPED; + + /* set non-incremental addressing mode */ + if (engine->non_incr_addr) + w |= (u32)XDMA_CTRL_NON_INCR_ADDR; + } + + dbg_tfr("iowrite32(0x%08x to 0x%p) (control)\n", w, + (void *)&engine->regs->control); + /* start the engine */ + write_register(w, &engine->regs->control, + (unsigned long)(&engine->regs->control) - + (unsigned long)(&engine->regs)); + + /* dummy read of status register to flush all previous writes */ + w = read_register(&engine->regs->status); + dbg_tfr("ioread32(0x%p) = 0x%08x (dummy read flushes writes).\n", + &engine->regs->status, w); +} + +/** + * engine_start() - start an idle engine with its first transfer on queue + * + * The engine will run and process all transfers that are queued using + * transfer_queue() and thus have their descriptor lists chained. + * + * During the run, new transfers will be processed if transfer_queue() has + * chained the descriptors before the hardware fetches the last descriptor. + * A transfer that was chained too late will invoke a new run of the engine + * initiated from the engine_service() routine. + * + * The engine must be idle and at least one transfer must be queued. + * This function does not take locks; the engine spinlock must already be + * taken. + * + */ +static struct xdma_transfer *engine_start(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer; + u32 w; + int extra_adj = 0; + + /* engine must be idle */ + BUG_ON(engine->running); + /* engine transfer queue must not be empty */ + BUG_ON(list_empty(&engine->transfer_list)); + /* inspect first transfer queued on the engine */ + transfer = list_entry(engine->transfer_list.next, struct xdma_transfer, + entry); + BUG_ON(!transfer); + + /* engine is no longer shutdown */ + engine->shutdown = ENGINE_SHUTDOWN_NONE; + + dbg_tfr("engine_start(%s): transfer=0x%p.\n", engine->name, transfer); + + /* initialize number of descriptors of dequeued transfers */ + engine->desc_dequeued = 0; + + /* write lower 32-bit of bus address of transfer first descriptor */ + w = cpu_to_le32(PCI_DMA_L(transfer->desc_bus)); + dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_lo)\n", w, + (void *)&engine->sgdma_regs->first_desc_lo); + write_register(w, &engine->sgdma_regs->first_desc_lo, + (unsigned long)(&engine->sgdma_regs->first_desc_lo) - + (unsigned long)(&engine->sgdma_regs)); + /* write upper 32-bit of bus address of transfer first descriptor */ + w = cpu_to_le32(PCI_DMA_H(transfer->desc_bus)); + dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_hi)\n", w, + (void *)&engine->sgdma_regs->first_desc_hi); + write_register(w, &engine->sgdma_regs->first_desc_hi, + (unsigned long)(&engine->sgdma_regs->first_desc_hi) - + (unsigned long)(&engine->sgdma_regs)); + + if (transfer->desc_adjacent > 0) { + extra_adj = transfer->desc_adjacent - 1; + if (extra_adj > MAX_EXTRA_ADJ) + extra_adj = MAX_EXTRA_ADJ; + } + dbg_tfr("iowrite32(0x%08x to 0x%p) (first_desc_adjacent)\n", + extra_adj, (void *)&engine->sgdma_regs->first_desc_adjacent); + write_register(extra_adj, &engine->sgdma_regs->first_desc_adjacent, + (unsigned long)(&engine->sgdma_regs->first_desc_adjacent) - + (unsigned long)(&engine->sgdma_regs)); + + dbg_tfr("ioread32(0x%p) (dummy read flushes writes).\n", + &engine->regs->status); + + #if LINUX_VERSION_CODE < KERNEL_VERSION(5,3,0) + mmiowb(); + #endif + + engine_start_mode_config(engine); + + engine_status_read(engine, 0, 0); + + dbg_tfr("%s engine 0x%p now running\n", engine->name, engine); + /* remember the engine is running */ + engine->running = 1; + return transfer; +} + +/** + * engine_service() - service an SG DMA engine + * + * must be called with engine->lock already acquired + * + * @engine pointer to struct xdma_engine + * + */ +static void engine_service_shutdown(struct xdma_engine *engine) +{ + /* if the engine stopped with RUN still asserted, de-assert RUN now */ + dbg_tfr("engine just went idle, resetting RUN_STOP.\n"); + xdma_engine_stop(engine); + engine->running = 0; + + /* awake task on engine's shutdown wait queue */ + wake_up_interruptible(&engine->shutdown_wq); +} + +struct xdma_transfer *engine_transfer_completion(struct xdma_engine *engine, + struct xdma_transfer *transfer) +{ + BUG_ON(!engine); + + if (unlikely(!transfer)) { + pr_info("%s: xfer empty.\n", engine->name); + return NULL; + } + + /* synchronous I/O? */ + /* awake task on transfer's wait queue */ + wake_up_interruptible(&transfer->wq); + + return transfer; +} + +struct xdma_transfer *engine_service_transfer_list(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 *pdesc_completed) +{ + BUG_ON(!engine); + BUG_ON(!pdesc_completed); + + if (unlikely(!transfer)) { + pr_info("%s xfer empty, pdesc completed %u.\n", + engine->name, *pdesc_completed); + return NULL; + } + + /* + * iterate over all the transfers completed by the engine, + * except for the last (i.e. use > instead of >=). + */ + while (transfer && (!transfer->cyclic) && + (*pdesc_completed > transfer->desc_num)) { + /* remove this transfer from pdesc_completed */ + *pdesc_completed -= transfer->desc_num; + dbg_tfr("%s engine completed non-cyclic xfer 0x%p (%d desc)\n", + engine->name, transfer, transfer->desc_num); + /* remove completed transfer from list */ + list_del(engine->transfer_list.next); + /* add to dequeued number of descriptors during this run */ + engine->desc_dequeued += transfer->desc_num; + /* mark transfer as succesfully completed */ + transfer->state = TRANSFER_STATE_COMPLETED; + + /* Complete transfer - sets transfer to NULL if an async + * transfer has completed */ + transfer = engine_transfer_completion(engine, transfer); + + /* if exists, get the next transfer on the list */ + if (!list_empty(&engine->transfer_list)) { + transfer = list_entry(engine->transfer_list.next, + struct xdma_transfer, entry); + dbg_tfr("Non-completed transfer %p\n", transfer); + } else { + /* no further transfers? */ + transfer = NULL; + } + } + + return transfer; +} + +static void engine_err_handle(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 desc_completed) +{ + u32 value; + + /* + * The BUSY bit is expected to be clear now but older HW has a race + * condition which could cause it to be still set. If it's set, re-read + * and check again. If it's still set, log the issue. + */ + if (engine->status & XDMA_STAT_BUSY) { + value = read_register(&engine->regs->status); + if ((value & XDMA_STAT_BUSY) && printk_ratelimit()) + pr_info("%s has errors but is still BUSY\n", + engine->name); + } + + if (printk_ratelimit()) { + pr_info("%s, s 0x%x, aborted xfer 0x%p, cmpl %d/%d\n", + engine->name, engine->status, transfer, desc_completed, + transfer->desc_num); + } + + /* mark transfer as failed */ + transfer->state = TRANSFER_STATE_FAILED; + xdma_engine_stop(engine); +} + +struct xdma_transfer *engine_service_final_transfer(struct xdma_engine *engine, + struct xdma_transfer *transfer, u32 *pdesc_completed) +{ + BUG_ON(!engine); + BUG_ON(!pdesc_completed); + + /* inspect the current transfer */ + if (unlikely(!transfer)) { + pr_info("%s xfer empty, pdesc completed %u.\n", + engine->name, *pdesc_completed); + return NULL; + } else { + if (((engine->dir == DMA_FROM_DEVICE) && + (engine->status & XDMA_STAT_C2H_ERR_MASK)) || + ((engine->dir == DMA_TO_DEVICE) && + (engine->status & XDMA_STAT_H2C_ERR_MASK))) { + pr_info("engine %s, status error 0x%x.\n", + engine->name, engine->status); + engine_status_dump(engine); + engine_err_handle(engine, transfer, *pdesc_completed); + goto transfer_del; + } + + if (engine->status & XDMA_STAT_BUSY) + pr_debug("engine %s is unexpectedly busy - ignoring\n", + engine->name); + + /* the engine stopped on current transfer? */ + if (*pdesc_completed < transfer->desc_num) { + transfer->state = TRANSFER_STATE_FAILED; + pr_info("%s, xfer 0x%p, stopped half-way, %d/%d.\n", + engine->name, transfer, *pdesc_completed, + transfer->desc_num); + } else { + dbg_tfr("engine %s completed transfer\n", engine->name); + dbg_tfr("Completed transfer ID = 0x%p\n", transfer); + dbg_tfr("*pdesc_completed=%d, transfer->desc_num=%d", + *pdesc_completed, transfer->desc_num); + + if (!transfer->cyclic) { + /* + * if the engine stopped on this transfer, + * it should be the last + */ + WARN_ON(*pdesc_completed > transfer->desc_num); + } + /* mark transfer as succesfully completed */ + transfer->state = TRANSFER_STATE_COMPLETED; + } + +transfer_del: + /* remove completed transfer from list */ + list_del(engine->transfer_list.next); + /* add to dequeued number of descriptors during this run */ + engine->desc_dequeued += transfer->desc_num; + + /* + * Complete transfer - sets transfer to NULL if an asynchronous + * transfer has completed + */ + transfer = engine_transfer_completion(engine, transfer); + } + + return transfer; +} + +static void engine_service_perf(struct xdma_engine *engine, u32 desc_completed) +{ + BUG_ON(!engine); + + /* performance measurement is running? */ + if (engine->xdma_perf) { + /* a descriptor was completed? */ + if (engine->status & XDMA_STAT_DESC_COMPLETED) { + engine->xdma_perf->iterations = desc_completed; + dbg_perf("transfer->xdma_perf->iterations=%d\n", + engine->xdma_perf->iterations); + } + + /* a descriptor stopped the engine? */ + if (engine->status & XDMA_STAT_DESC_STOPPED) { + engine->xdma_perf->stopped = 1; + /* + * wake any XDMA_PERF_IOCTL_STOP waiting for + * the performance run to finish + */ + wake_up_interruptible(&engine->xdma_perf_wq); + dbg_perf("transfer->xdma_perf stopped\n"); + } + } +} + +static void engine_transfer_dequeue(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer; + + BUG_ON(!engine); + + /* pick first transfer on the queue (was submitted to the engine) */ + transfer = list_entry(engine->transfer_list.next, struct xdma_transfer, + entry); + if (!transfer || transfer != &engine->cyclic_req->xfer) { + pr_info("%s, xfer 0x%p != 0x%p.\n", + engine->name, transfer, &engine->cyclic_req->xfer); + return; + } + dbg_tfr("%s engine completed cyclic transfer 0x%p (%d desc).\n", + engine->name, transfer, transfer->desc_num); + /* remove completed transfer from list */ + list_del(engine->transfer_list.next); +} + +static int engine_ring_process(struct xdma_engine *engine) +{ + struct xdma_result *result; + int start; + int eop_count = 0; + + BUG_ON(!engine); + result = engine->cyclic_result; + BUG_ON(!result); + + /* where we start receiving in the ring buffer */ + start = engine->rx_tail; + + /* iterate through all newly received RX result descriptors */ + dbg_tfr("%s, result %d, 0x%x, len 0x%x.\n", + engine->name, engine->rx_tail, result[engine->rx_tail].status, + result[engine->rx_tail].length); + while (result[engine->rx_tail].status && !engine->rx_overrun) { + /* EOP bit set in result? */ + if (result[engine->rx_tail].status & RX_STATUS_EOP){ + eop_count++; + } + + /* increment tail pointer */ + engine->rx_tail = (engine->rx_tail + 1) % CYCLIC_RX_PAGES_MAX; + + dbg_tfr("%s, head %d, tail %d, 0x%x, len 0x%x.\n", + engine->name, engine->rx_head, engine->rx_tail, + result[engine->rx_tail].status, + result[engine->rx_tail].length); + + /* overrun? */ + if (engine->rx_tail == engine->rx_head) { + dbg_tfr("%s: overrun\n", engine->name); + /* flag to user space that overrun has occurred */ + engine->rx_overrun = 1; + } + } + + return eop_count; +} + +static int engine_service_cyclic_polled(struct xdma_engine *engine) +{ + int eop_count = 0; + int rc = 0; + struct xdma_poll_wb *writeback_data; + u32 sched_limit = 0; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + writeback_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + + while (eop_count == 0) { + if (sched_limit != 0) { + if ((sched_limit % NUM_POLLS_PER_SCHED) == 0) + schedule(); + } + sched_limit++; + + /* Monitor descriptor writeback address for errors */ + if ((writeback_data->completed_desc_count) & WB_ERR_MASK) { + rc = -1; + break; + } + + eop_count = engine_ring_process(engine); + } + + if (eop_count == 0) { + engine_status_read(engine, 1, 0); + if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) + engine_transfer_dequeue(engine); + + engine_service_shutdown(engine); + } + } + + return rc; +} + +static int engine_service_cyclic_interrupt(struct xdma_engine *engine) +{ + int eop_count = 0; + struct xdma_transfer *xfer; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + engine_status_read(engine, 1, 0); + + eop_count = engine_ring_process(engine); + /* + * wake any reader on EOP, as one or more packets are now in + * the RX buffer + */ + xfer = &engine->cyclic_req->xfer; + if(enable_credit_mp){ + if (eop_count > 0) { + //engine->eop_found = 1; + } + wake_up_interruptible(&xfer->wq); + }else{ + if (eop_count > 0) { + /* awake task on transfer's wait queue */ + dbg_tfr("wake_up_interruptible() due to %d EOP's\n", eop_count); + engine->eop_found = 1; + wake_up_interruptible(&xfer->wq); + } + } + + /* engine was running but is no longer busy? */ + if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) + engine_transfer_dequeue(engine); + + engine_service_shutdown(engine); + } + + return 0; +} + +/* must be called with engine->lock already acquired */ +static int engine_service_cyclic(struct xdma_engine *engine) +{ + int rc = 0; + + dbg_tfr("engine_service_cyclic()"); + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + if (poll_mode) + rc = engine_service_cyclic_polled(engine); + else + rc = engine_service_cyclic_interrupt(engine); + + return rc; +} + + +static void engine_service_resume(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer_started; + + BUG_ON(!engine); + + /* engine stopped? */ + if (!engine->running) { + /* in the case of shutdown, let it finish what's in the Q */ + if (!list_empty(&engine->transfer_list)) { + /* (re)start engine */ + transfer_started = engine_start(engine); + pr_info("re-started %s engine with pending xfer 0x%p\n", + engine->name, transfer_started); + /* engine was requested to be shutdown? */ + } else if (engine->shutdown & ENGINE_SHUTDOWN_REQUEST) { + engine->shutdown |= ENGINE_SHUTDOWN_IDLE; + /* awake task on engine's shutdown wait queue */ + wake_up_interruptible(&engine->shutdown_wq); + } else { + dbg_tfr("no pending transfers, %s engine stays idle.\n", + engine->name); + } + } else { + /* engine is still running? */ + if (list_empty(&engine->transfer_list)) { + pr_warn("no queued transfers but %s engine running!\n", + engine->name); + WARN_ON(1); + } + } +} + +/** + * engine_service() - service an SG DMA engine + * + * must be called with engine->lock already acquired + * + * @engine pointer to struct xdma_engine + * + */ +static int engine_service(struct xdma_engine *engine, int desc_writeback) +{ + struct xdma_transfer *transfer = NULL; + u32 desc_count = desc_writeback & WB_COUNT_MASK; + u32 err_flag = desc_writeback & WB_ERR_MASK; + int rv = 0; + struct xdma_poll_wb *wb_data; + + BUG_ON(!engine); + + /* If polling detected an error, signal to the caller */ + if (err_flag) + rv = -1; + + /* Service the engine */ + if (!engine->running) { + dbg_tfr("Engine was not running!!! Clearing status\n"); + engine_status_read(engine, 1, 0); + return 0; + } + + /* + * If called by the ISR or polling detected an error, read and clear + * engine status. For polled mode descriptor completion, this read is + * unnecessary and is skipped to reduce latency + */ + if ((desc_count == 0) || (err_flag != 0)) + engine_status_read(engine, 1, 0); + + /* + * engine was running but is no longer busy, or writeback occurred, + * shut down + */ + if ((engine->running && !(engine->status & XDMA_STAT_BUSY)) || + (desc_count != 0)) + engine_service_shutdown(engine); + + /* + * If called from the ISR, or if an error occurred, the descriptor + * count will be zero. In this scenario, read the descriptor count + * from HW. In polled mode descriptor completion, this read is + * unnecessary and is skipped to reduce latency + */ + if (!desc_count) + desc_count = read_register(&engine->regs->completed_desc_count); + dbg_tfr("desc_count = %d\n", desc_count); + + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) { + /* pick first transfer on queue (was submitted to the engine) */ + transfer = list_entry(engine->transfer_list.next, + struct xdma_transfer, entry); + + dbg_tfr("head of queue transfer 0x%p has %d descriptors\n", + transfer, (int)transfer->desc_num); + + dbg_tfr("Engine completed %d desc, %d not yet dequeued\n", + (int)desc_count, + (int)desc_count - engine->desc_dequeued); + + engine_service_perf(engine, desc_count); + } + + /* account for already dequeued transfers during this engine run */ + desc_count -= engine->desc_dequeued; + + /* Process all but the last transfer */ + transfer = engine_service_transfer_list(engine, transfer, &desc_count); + + /* + * Process final transfer - includes checks of number of descriptors to + * detect faulty completion + */ + transfer = engine_service_final_transfer(engine, transfer, &desc_count); + + /* Before starting engine again, clear the writeback data */ + if (poll_mode) { + wb_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + wb_data->completed_desc_count = 0; + } + + /* Restart the engine following the servicing */ + engine_service_resume(engine); + + return 0; +} + +/* engine_service_work */ +static void engine_service_work(struct work_struct *work) +{ + struct xdma_engine *engine; + unsigned long flags; + + engine = container_of(work, struct xdma_engine, work); + BUG_ON(engine->magic != MAGIC_ENGINE); + + /* lock the engine */ + spin_lock_irqsave(&engine->lock, flags); + + dbg_tfr("engine_service() for %s engine %p\n", + engine->name, engine); + if (engine->cyclic_req) + engine_service_cyclic(engine); + else + engine_service(engine, 0); + + /* re-enable interrupts for this engine */ + if (engine->xdev->msix_enabled){ + write_register(engine->interrupt_enable_mask_value, + &engine->regs->interrupt_enable_mask_w1s, + (unsigned long)(&engine->regs->interrupt_enable_mask_w1s) - + (unsigned long)(&engine->regs)); + } else + channel_interrupts_enable(engine->xdev, engine->irq_bitmask); + + /* unlock the engine */ + spin_unlock_irqrestore(&engine->lock, flags); +} + +static u32 engine_service_wb_monitor(struct xdma_engine *engine, + u32 expected_wb) +{ + struct xdma_poll_wb *wb_data; + u32 desc_wb = 0; + u32 sched_limit = 0; + unsigned long timeout; + + BUG_ON(!engine); + wb_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + + /* + * Poll the writeback location for the expected number of + * descriptors / error events This loop is skipped for cyclic mode, + * where the expected_desc_count passed in is zero, since it cannot be + * determined before the function is called + */ + + timeout = jiffies + (POLL_TIMEOUT_SECONDS * HZ); + while (expected_wb != 0) { + desc_wb = wb_data->completed_desc_count; + + if (desc_wb & WB_ERR_MASK) + break; + else if (desc_wb == expected_wb) + break; + + /* RTO - prevent system from hanging in polled mode */ + if (time_after(jiffies, timeout)) { + dbg_tfr("Polling timeout occurred"); + dbg_tfr("desc_wb = 0x%08x, expected 0x%08x\n", desc_wb, + expected_wb); + if ((desc_wb & WB_COUNT_MASK) > expected_wb) + desc_wb = expected_wb | WB_ERR_MASK; + + break; + } + + /* + * Define NUM_POLLS_PER_SCHED to limit how much time is spent + * in the scheduler + */ + + if (sched_limit != 0) { + if ((sched_limit % NUM_POLLS_PER_SCHED) == 0) + schedule(); + } + sched_limit++; + } + + return desc_wb; +} + +static int engine_service_poll(struct xdma_engine *engine, + u32 expected_desc_count) +{ + struct xdma_poll_wb *writeback_data; + u32 desc_wb = 0; + unsigned long flags; + int rv = 0; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + writeback_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + + if ((expected_desc_count & WB_COUNT_MASK) != expected_desc_count) { + dbg_tfr("Queued descriptor count is larger than supported\n"); + return -1; + } + + /* + * Poll the writeback location for the expected number of + * descriptors / error events This loop is skipped for cyclic mode, + * where the expected_desc_count passed in is zero, since it cannot be + * determined before the function is called + */ + + desc_wb = engine_service_wb_monitor(engine, expected_desc_count); + + spin_lock_irqsave(&engine->lock, flags); + dbg_tfr("%s service.\n", engine->name); + if (engine->cyclic_req) { + rv = engine_service_cyclic(engine); + } else { + rv = engine_service(engine, desc_wb); + } + spin_unlock_irqrestore(&engine->lock, flags); + + return rv; +} + +static irqreturn_t user_irq_service(int irq, struct xdma_user_irq *user_irq) +{ + unsigned long flags; + + BUG_ON(!user_irq); + + if (user_irq->handler) + return user_irq->handler(user_irq->user_idx, user_irq->dev); + + spin_lock_irqsave(&(user_irq->events_lock), flags); + if (!user_irq->events_irq) { + user_irq->events_irq = 1; + wake_up_interruptible(&(user_irq->events_wq)); + } + spin_unlock_irqrestore(&(user_irq->events_lock), flags); + + return IRQ_HANDLED; +} + +/* + * xdma_isr() - Interrupt handler + * + * @dev_id pointer to xdma_dev + */ +static irqreturn_t xdma_isr(int irq, void *dev_id) +{ + u32 ch_irq; + u32 user_irq; + u32 mask; + struct xdma_dev *xdev; + struct interrupt_regs *irq_regs; + + dbg_irq("(irq=%d, dev 0x%p) <<<< ISR.\n", irq, dev_id); + BUG_ON(!dev_id); + xdev = (struct xdma_dev *)dev_id; + + if (!xdev) { + WARN_ON(!xdev); + dbg_irq("xdma_isr(irq=%d) xdev=%p ??\n", irq, xdev); + return IRQ_NONE; + } + + irq_regs = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + + /* read channel interrupt requests */ + ch_irq = read_register(&irq_regs->channel_int_request); + dbg_irq("ch_irq = 0x%08x\n", ch_irq); + + /* + * disable all interrupts that fired; these are re-enabled individually + * after the causing module has been fully serviced. + */ + if (ch_irq) + channel_interrupts_disable(xdev, ch_irq); + + /* read user interrupts - this read also flushes the above write */ + user_irq = read_register(&irq_regs->user_int_request); + dbg_irq("user_irq = 0x%08x\n", user_irq); + + if (user_irq) { + int user = 0; + u32 mask = 1; + int max = xdev->h2c_channel_max; + + for (; user < max && user_irq; user++, mask <<= 1) { + if (user_irq & mask) { + user_irq &= ~mask; + user_irq_service(irq, &xdev->user_irq[user]); + } + } + } + + mask = ch_irq & xdev->mask_irq_h2c; + if (mask) { + int channel = 0; + int max = xdev->h2c_channel_max; + + /* iterate over H2C (PCIe read) */ + for (channel = 0; channel < max && mask; channel++) { + struct xdma_engine *engine = &xdev->engine_h2c[channel]; + + /* engine present and its interrupt fired? */ + if((engine->irq_bitmask & mask) && + (engine->magic == MAGIC_ENGINE)) { + mask &= ~engine->irq_bitmask; + dbg_tfr("schedule_work, %s.\n", engine->name); + schedule_work(&engine->work); + } + } + } + + mask = ch_irq & xdev->mask_irq_c2h; + if (mask) { + int channel = 0; + int max = xdev->c2h_channel_max; + + /* iterate over C2H (PCIe write) */ + for (channel = 0; channel < max && mask; channel++) { + struct xdma_engine *engine = &xdev->engine_c2h[channel]; + + /* engine present and its interrupt fired? */ + if((engine->irq_bitmask & mask) && + (engine->magic == MAGIC_ENGINE)) { + mask &= ~engine->irq_bitmask; + dbg_tfr("schedule_work, %s.\n", engine->name); + schedule_work(&engine->work); + } + } + } + + xdev->irq_count++; + return IRQ_HANDLED; +} + +/* + * xdma_user_irq() - Interrupt handler for user interrupts in MSI-X mode + * + * @dev_id pointer to xdma_dev + */ +static irqreturn_t xdma_user_irq(int irq, void *dev_id) +{ + struct xdma_user_irq *user_irq; + + dbg_irq("(irq=%d) <<<< INTERRUPT SERVICE ROUTINE\n", irq); + + BUG_ON(!dev_id); + user_irq = (struct xdma_user_irq *)dev_id; + + return user_irq_service(irq, user_irq); +} + +/* + * xdma_channel_irq() - Interrupt handler for channel interrupts in MSI-X mode + * + * @dev_id pointer to xdma_dev + */ +static irqreturn_t xdma_channel_irq(int irq, void *dev_id) +{ + struct xdma_dev *xdev; + struct xdma_engine *engine; + struct interrupt_regs *irq_regs; + + dbg_irq("(irq=%d) <<<< INTERRUPT service ROUTINE\n", irq); + BUG_ON(!dev_id); + + engine = (struct xdma_engine *)dev_id; + xdev = engine->xdev; + + if (!xdev) { + WARN_ON(!xdev); + dbg_irq("xdma_channel_irq(irq=%d) xdev=%p ??\n", irq, xdev); + return IRQ_NONE; + } + + irq_regs = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + + /* Disable the interrupt for this engine */ + write_register(engine->interrupt_enable_mask_value, + &engine->regs->interrupt_enable_mask_w1c, + (unsigned long) + (&engine->regs->interrupt_enable_mask_w1c) - + (unsigned long)(&engine->regs)); + /* Dummy read to flush the above write */ + read_register(&irq_regs->channel_int_pending); + /* Schedule the bottom half */ + schedule_work(&engine->work); + + /* + * RTO - need to protect access here if multiple MSI-X are used for + * user interrupts + */ + xdev->irq_count++; + return IRQ_HANDLED; +} + +/* + * Unmap the BAR regions that had been mapped earlier using map_bars() + */ +static void unmap_bars(struct xdma_dev *xdev, struct pci_dev *dev) +{ + int i; + + for (i = 0; i < XDMA_BAR_NUM; i++) { + /* is this BAR mapped? */ + if (xdev->bar[i]) { + /* unmap BAR */ + pci_iounmap(dev, xdev->bar[i]); + /* mark as unmapped */ + xdev->bar[i] = NULL; + } + } +} + +static int map_single_bar(struct xdma_dev *xdev, struct pci_dev *dev, int idx) +{ + resource_size_t bar_start; + resource_size_t bar_len; + resource_size_t map_len; + + bar_start = pci_resource_start(dev, idx); + bar_len = pci_resource_len(dev, idx); + map_len = bar_len; + + xdev->bar[idx] = NULL; + + /* do not map BARs with length 0. Note that start MAY be 0! */ + if (!bar_len) { + //pr_info("BAR #%d is not present - skipping\n", idx); + return 0; + } + + /* BAR size exceeds maximum desired mapping? */ + if (bar_len > INT_MAX) { + pr_info("Limit BAR %d mapping from %llu to %d bytes\n", idx, + (u64)bar_len, INT_MAX); + map_len = (resource_size_t)INT_MAX; + } + /* + * map the full device memory or IO region into kernel virtual + * address space + */ + dbg_init("BAR%d: %llu bytes to be mapped.\n", idx, (u64)map_len); + xdev->bar[idx] = pci_iomap(dev, idx, map_len); + + if (!xdev->bar[idx]) { + pr_info("Could not map BAR %d.\n", idx); + return -1; + } + + pr_info("BAR%d at 0x%llx mapped at 0x%p, length=%llu(/%llu)\n", idx, + (u64)bar_start, xdev->bar[idx], (u64)map_len, (u64)bar_len); + + return (int)map_len; +} + +static int is_config_bar(struct xdma_dev *xdev, int idx) +{ + u32 irq_id = 0; + u32 cfg_id = 0; + int flag = 0; + u32 mask = 0xffff0000; /* Compare only XDMA ID's not Version number */ + struct interrupt_regs *irq_regs = + (struct interrupt_regs *) (xdev->bar[idx] + XDMA_OFS_INT_CTRL); + struct config_regs *cfg_regs = + (struct config_regs *)(xdev->bar[idx] + XDMA_OFS_CONFIG); + + irq_id = read_register(&irq_regs->identifier); + cfg_id = read_register(&cfg_regs->identifier); + + if (((irq_id & mask)== IRQ_BLOCK_ID) && + ((cfg_id & mask)== CONFIG_BLOCK_ID)) { + dbg_init("BAR %d is the XDMA config BAR\n", idx); + flag = 1; + } else { + dbg_init("BAR %d is NOT the XDMA config BAR: 0x%x, 0x%x.\n", + idx, irq_id, cfg_id); + flag = 0; + } + + return flag; +} + +static void identify_bars(struct xdma_dev *xdev, int *bar_id_list, int num_bars, + int config_bar_pos) +{ + /* + * The following logic identifies which BARs contain what functionality + * based on the position of the XDMA config BAR and the number of BARs + * detected. The rules are that the user logic and bypass logic BARs + * are optional. When both are present, the XDMA config BAR will be the + * 2nd BAR detected (config_bar_pos = 1), with the user logic being + * detected first and the bypass being detected last. When one is + * omitted, the type of BAR present can be identified by whether the + * XDMA config BAR is detected first or last. When both are omitted, + * only the XDMA config BAR is present. This somewhat convoluted + * approach is used instead of relying on BAR numbers in order to work + * correctly with both 32-bit and 64-bit BARs. + */ + + BUG_ON(!xdev); + BUG_ON(!bar_id_list); + + dbg_init("xdev 0x%p, bars %d, config at %d.\n", + xdev, num_bars, config_bar_pos); + + switch (num_bars) { + case 1: + /* Only one BAR present - no extra work necessary */ + break; + + case 2: + if (config_bar_pos == 0) { + xdev->bypass_bar_idx = bar_id_list[1]; + } else if (config_bar_pos == 1) { + xdev->user_bar_idx = bar_id_list[0]; + } else { + pr_info("2, XDMA config BAR unexpected %d.\n", + config_bar_pos); + } + break; + + case 3: + case 4: + if ((config_bar_pos == 1) || (config_bar_pos == 2)) { + /* user bar at bar #0 */ + xdev->user_bar_idx = bar_id_list[0]; + /* bypass bar at the last bar */ + xdev->bypass_bar_idx = bar_id_list[num_bars - 1]; + } else { + pr_info("3/4, XDMA config BAR unexpected %d.\n", + config_bar_pos); + } + break; + + default: + /* Should not occur - warn user but safe to continue */ + pr_info("Unexpected # BARs (%d), XDMA config BAR only.\n", + num_bars); + break; + + } + pr_info("%d BARs: config %d, user %d, bypass %d.\n", + num_bars, config_bar_pos, xdev->user_bar_idx, + xdev->bypass_bar_idx); +} + +/* map_bars() -- map device regions into kernel virtual address space + * + * Map the device memory regions into kernel virtual address space after + * verifying their sizes respect the minimum sizes needed + */ +static int map_bars(struct xdma_dev *xdev, struct pci_dev *dev) +{ + int rv; + int i; + int bar_id_list[XDMA_BAR_NUM]; + int bar_id_idx = 0; + int config_bar_pos = 0; + + /* iterate through all the BARs */ + for (i = 0; i < XDMA_BAR_NUM; i++) { + int bar_len; + + bar_len = map_single_bar(xdev, dev, i); + if (bar_len == 0) { + continue; + } else if (bar_len < 0) { + rv = -EINVAL; + goto fail; + } + + /* Try to identify BAR as XDMA control BAR */ + if ((bar_len >= XDMA_BAR_SIZE) && (xdev->config_bar_idx < 0)) { + + if (is_config_bar(xdev, i)) { + xdev->config_bar_idx = i; + config_bar_pos = bar_id_idx; + pr_info("config bar %d, pos %d.\n", + xdev->config_bar_idx, config_bar_pos); + } + } + + bar_id_list[bar_id_idx] = i; + bar_id_idx++; + } + + /* The XDMA config BAR must always be present */ + if (xdev->config_bar_idx < 0) { + pr_info("Failed to detect XDMA config BAR\n"); + rv = -EINVAL; + goto fail; + } + +#ifdef __LIBXDMA_CONFIG_BAR_ONLY__ + /* unmapped all other bars, except XDMA config. bar */ + for (i = 0; i < XDMA_BAR_NUM; i++) { + if (i == xdev->config_bar_idx) + continue; + + /* is this BAR mapped? */ + if (xdev->bar[i]) { + /* unmap BAR */ + pci_iounmap(dev, xdev->bar[i]); + /* mark as unmapped */ + xdev->bar[i] = NULL; + pr_info("unmap non-config bar %d.\n", i); + } + } +#else + identify_bars(xdev, bar_id_list, bar_id_idx, config_bar_pos); +#endif + + /* successfully mapped all required BAR regions */ + return 0; + +fail: + /* unwind; unmap any BARs that we did map */ + unmap_bars(xdev, dev); + return rv; +} + +/* + * MSI-X interrupt: + * <h2c+c2h channel_max> vectors, followed by <user_max> vectors + */ + +/* + * RTO - code to detect if MSI/MSI-X capability exists is derived + * from linux/pci/msi.c - pci_msi_check_device + */ + +#ifndef arch_msi_check_device +int arch_msi_check_device(struct pci_dev *dev, int nvec, int type) +{ + return 0; +} +#endif + +/* type = PCI_CAP_ID_MSI or PCI_CAP_ID_MSIX */ +static int msi_msix_capable(struct pci_dev *dev, int type) +{ + struct pci_bus *bus; + int ret; + + if (!dev || dev->no_msi) + return 0; + + for (bus = dev->bus; bus; bus = bus->parent) + if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) + return 0; + + ret = arch_msi_check_device(dev, 1, type); + if (ret) + return 0; + + if (!pci_find_capability(dev, type)) + return 0; + + return 1; +} + +static void disable_msi_msix(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + if (xdev->msix_enabled) { + pci_disable_msix(pdev); + xdev->msix_enabled = 0; + } else if (xdev->msi_enabled) { + pci_disable_msi(pdev); + xdev->msi_enabled = 0; + } +} + +static int enable_msi_msix(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + int rv = 0; + + BUG_ON(!xdev); + BUG_ON(!pdev); + + if (!interrupt_mode && msi_msix_capable(pdev, PCI_CAP_ID_MSIX)) { + int req_nvec = xdev->c2h_channel_max + xdev->h2c_channel_max + + xdev->user_max; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + dbg_init("Enabling MSI-X\n"); + rv = pci_alloc_irq_vectors(pdev, req_nvec, req_nvec, + PCI_IRQ_MSIX); +#else + int i; + + dbg_init("Enabling MSI-X\n"); + for (i = 0; i < req_nvec; i++) + xdev->entry[i].entry = i; + + rv = pci_enable_msix(pdev, xdev->entry, req_nvec); +#endif + if (rv < 0) + dbg_init("Couldn't enable MSI-X mode: %d\n", rv); + + xdev->msix_enabled = 1; + + } else if (interrupt_mode == 1 && + msi_msix_capable(pdev, PCI_CAP_ID_MSI)) { + /* enable message signalled interrupts */ + dbg_init("pci_enable_msi()\n"); + rv = pci_enable_msi(pdev); + if (rv < 0) + dbg_init("Couldn't enable MSI mode: %d\n", rv); + xdev->msi_enabled = 1; + + } else { + dbg_init("MSI/MSI-X not detected - using legacy interrupts\n"); + } + + return rv; +} + +static void pci_check_intr_pend(struct pci_dev *pdev) +{ + u16 v; + + pci_read_config_word(pdev, PCI_STATUS, &v); + if (v & PCI_STATUS_INTERRUPT) { + pr_info("%s PCI STATUS Interrupt pending 0x%x.\n", + dev_name(&pdev->dev), v); + pci_write_config_word(pdev, PCI_STATUS, PCI_STATUS_INTERRUPT); + } +} + +static void pci_keep_intx_enabled(struct pci_dev *pdev) +{ + /* workaround to a h/w bug: + * when msix/msi become unavaile, default to legacy. + * However the legacy enable was not checked. + * If the legacy was disabled, no ack then everything stuck + */ + u16 pcmd, pcmd_new; + + pci_read_config_word(pdev, PCI_COMMAND, &pcmd); + pcmd_new = pcmd & ~PCI_COMMAND_INTX_DISABLE; + if (pcmd_new != pcmd) { + pr_info("%s: clear INTX_DISABLE, 0x%x -> 0x%x.\n", + dev_name(&pdev->dev), pcmd, pcmd_new); + pci_write_config_word(pdev, PCI_COMMAND, pcmd_new); + } +} + +static void prog_irq_msix_user(struct xdma_dev *xdev, bool clear) +{ + /* user */ + struct interrupt_regs *int_regs = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + u32 i = xdev->c2h_channel_max + xdev->h2c_channel_max; + u32 max = i + xdev->user_max; + int j; + + for (j = 0; i < max; j++) { + u32 val = 0; + int k; + int shift = 0; + + if (clear) + i += 4; + else + for (k = 0; k < 4 && i < max; i++, k++, shift += 8) + val |= (i & 0x1f) << shift; + + write_register(val, &int_regs->user_msi_vector[j], + XDMA_OFS_INT_CTRL + + ((unsigned long)&int_regs->user_msi_vector[j] - + (unsigned long)int_regs)); + + dbg_init("vector %d, 0x%x.\n", j, val); + } +} + +static void prog_irq_msix_channel(struct xdma_dev *xdev, bool clear) +{ + struct interrupt_regs *int_regs = (struct interrupt_regs *) + (xdev->bar[xdev->config_bar_idx] + + XDMA_OFS_INT_CTRL); + u32 max = xdev->c2h_channel_max + xdev->h2c_channel_max; + u32 i; + int j; + + /* engine */ + for (i = 0, j = 0; i < max; j++) { + u32 val = 0; + int k; + int shift = 0; + + if (clear) + i += 4; + else + for (k = 0; k < 4 && i < max; i++, k++, shift += 8) + val |= (i & 0x1f) << shift; + + write_register(val, &int_regs->channel_msi_vector[j], + XDMA_OFS_INT_CTRL + + ((unsigned long)&int_regs->channel_msi_vector[j] - + (unsigned long)int_regs)); + dbg_init("vector %d, 0x%x.\n", j, val); + } +} + +static void irq_msix_channel_teardown(struct xdma_dev *xdev) +{ + struct xdma_engine *engine; + int j = 0; + int i = 0; + + if (!xdev->msix_enabled) + return; + + prog_irq_msix_channel(xdev, 1); + + engine = xdev->engine_h2c; + for (i = 0; i < xdev->h2c_channel_max; i++, j++, engine++) { + if (!engine->msix_irq_line) + break; + dbg_sg("Release IRQ#%d for engine %p\n", engine->msix_irq_line, + engine); + free_irq(engine->msix_irq_line, engine); + } + + engine = xdev->engine_c2h; + for (i = 0; i < xdev->c2h_channel_max; i++, j++, engine++) { + if (!engine->msix_irq_line) + break; + dbg_sg("Release IRQ#%d for engine %p\n", engine->msix_irq_line, + engine); + free_irq(engine->msix_irq_line, engine); + } +} + +static int irq_msix_channel_setup(struct xdma_dev *xdev) +{ + int i; + int j = xdev->h2c_channel_max; + int rv = 0; + u32 vector; + struct xdma_engine *engine; + + BUG_ON(!xdev); + if (!xdev->msix_enabled) + return 0; + + engine = xdev->engine_h2c; + for (i = 0; i < xdev->h2c_channel_max; i++, engine++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + vector = pci_irq_vector(xdev->pdev, i); +#else + vector = xdev->entry[i].vector; +#endif + rv = request_irq(vector, xdma_channel_irq, 0, xdev->mod_name, + engine); + if (rv) { + pr_info("requesti irq#%d failed %d, engine %s.\n", + vector, rv, engine->name); + return rv; + } + pr_info("engine %s, irq#%d.\n", engine->name, vector); + engine->msix_irq_line = vector; + } + + engine = xdev->engine_c2h; + for (i = 0; i < xdev->c2h_channel_max; i++, j++, engine++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + vector = pci_irq_vector(xdev->pdev, j); +#else + vector = xdev->entry[j].vector; +#endif + rv = request_irq(vector, xdma_channel_irq, 0, xdev->mod_name, + engine); + if (rv) { + pr_info("requesti irq#%d failed %d, engine %s.\n", + vector, rv, engine->name); + return rv; + } + pr_info("engine %s, irq#%d.\n", engine->name, vector); + engine->msix_irq_line = vector; + } + + return 0; +} + +static void irq_msix_user_teardown(struct xdma_dev *xdev) +{ + int i; + int j = xdev->h2c_channel_max + xdev->c2h_channel_max; + + BUG_ON(!xdev); + + if (!xdev->msix_enabled) + return; + + prog_irq_msix_user(xdev, 1); + + for (i = 0; i < xdev->user_max; i++, j++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + u32 vector = pci_irq_vector(xdev->pdev, j); +#else + u32 vector = xdev->entry[j].vector; +#endif + dbg_init("user %d, releasing IRQ#%d\n", i, vector); + free_irq(vector, &xdev->user_irq[i]); + } +} + +static int irq_msix_user_setup(struct xdma_dev *xdev) +{ + int i; + int j = xdev->h2c_channel_max + xdev->c2h_channel_max; + int rv = 0; + + /* vectors set in probe_scan_for_msi() */ + for (i = 0; i < xdev->user_max; i++, j++) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + u32 vector = pci_irq_vector(xdev->pdev, j); +#else + u32 vector = xdev->entry[j].vector; +#endif + rv = request_irq(vector, xdma_user_irq, 0, xdev->mod_name, + &xdev->user_irq[i]); + if (rv) { + pr_info("user %d couldn't use IRQ#%d, %d\n", + i, vector, rv); + break; + } + pr_info("%d-USR-%d, IRQ#%d with 0x%p\n", xdev->idx, i, vector, + &xdev->user_irq[i]); + } + + /* If any errors occur, free IRQs that were successfully requested */ + if (rv) { + for (i--, j--; i >= 0; i--, j--) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + u32 vector = pci_irq_vector(xdev->pdev, j); +#else + u32 vector = xdev->entry[j].vector; +#endif + free_irq(vector, &xdev->user_irq[i]); + } + } + + return rv; +} + +static int irq_msi_setup(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + int rv; + + xdev->irq_line = (int)pdev->irq; + rv = request_irq(pdev->irq, xdma_isr, 0, xdev->mod_name, xdev); + if (rv) + dbg_init("Couldn't use IRQ#%d, %d\n", pdev->irq, rv); + else + dbg_init("Using IRQ#%d with 0x%p\n", pdev->irq, xdev); + + return rv; +} + +static int irq_legacy_setup(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + u32 w; + u8 val; + void *reg; + int rv; + + pci_read_config_byte(pdev, PCI_INTERRUPT_PIN, &val); + dbg_init("Legacy Interrupt register value = %d\n", val); + if (val > 1) { + val--; + w = (val<<24) | (val<<16) | (val<<8)| val; + /* Program IRQ Block Channel vactor and IRQ Block User vector + * with Legacy interrupt value */ + reg = xdev->bar[xdev->config_bar_idx] + 0x2080; // IRQ user + write_register(w, reg, 0x2080); + write_register(w, reg+0x4, 0x2084); + write_register(w, reg+0x8, 0x2088); + write_register(w, reg+0xC, 0x208C); + reg = xdev->bar[xdev->config_bar_idx] + 0x20A0; // IRQ Block + write_register(w, reg, 0x20A0); + write_register(w, reg+0x4, 0x20A4); + } + + xdev->irq_line = (int)pdev->irq; + rv = request_irq(pdev->irq, xdma_isr, IRQF_SHARED, xdev->mod_name, + xdev); + if (rv) + dbg_init("Couldn't use IRQ#%d, %d\n", pdev->irq, rv); + else + dbg_init("Using IRQ#%d with 0x%p\n", pdev->irq, xdev); + + return rv; +} + +static void irq_teardown(struct xdma_dev *xdev) +{ + if (xdev->msix_enabled) { + irq_msix_channel_teardown(xdev); + irq_msix_user_teardown(xdev); + } else if (xdev->irq_line != -1) { + dbg_init("Releasing IRQ#%d\n", xdev->irq_line); + free_irq(xdev->irq_line, xdev); + } +} + +static int irq_setup(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + pci_keep_intx_enabled(pdev); + + if (xdev->msix_enabled) { + int rv = irq_msix_channel_setup(xdev); + if (rv) + return rv; + rv = irq_msix_user_setup(xdev); + if (rv) + return rv; + prog_irq_msix_channel(xdev, 0); + prog_irq_msix_user(xdev, 0); + + return 0; + } else if (xdev->msi_enabled) + return irq_msi_setup(xdev, pdev); + + return irq_legacy_setup(xdev, pdev); +} + +#ifdef __LIBXDMA_DEBUG__ +static void dump_desc(struct xdma_desc *desc_virt) +{ + int j; + u32 *p = (u32 *)desc_virt; + static char * const field_name[] = { + "magic|extra_adjacent|control", "bytes", "src_addr_lo", + "src_addr_hi", "dst_addr_lo", "dst_addr_hi", "next_addr", + "next_addr_pad"}; + char *dummy; + + /* remove warning about unused variable when debug printing is off */ + dummy = field_name[0]; + + for (j = 0; j < 8; j += 1) { + pr_info("0x%08lx/0x%02lx: 0x%08x 0x%08x %s\n", + (uintptr_t)p, (uintptr_t)p & 15, (int)*p, + le32_to_cpu(*p), field_name[j]); + p++; + } + pr_info("\n"); +} + +static void transfer_dump(struct xdma_transfer *transfer) +{ + int i; + struct xdma_desc *desc_virt = transfer->desc_virt; + + pr_info("xfer 0x%p, state 0x%x, f 0x%x, dir %d, len %u, last %d.\n", + transfer, transfer->state, transfer->flags, transfer->dir, + transfer->len, transfer->last_in_request); + + pr_info("transfer 0x%p, desc %d, bus 0x%llx, adj %d.\n", + transfer, transfer->desc_num, (u64)transfer->desc_bus, + transfer->desc_adjacent); + for (i = 0; i < transfer->desc_num; i += 1) + dump_desc(desc_virt + i); +} +#endif /* __LIBXDMA_DEBUG__ */ + +/* xdma_desc_alloc() - Allocate cache-coherent array of N descriptors. + * + * Allocates an array of 'number' descriptors in contiguous PCI bus addressable + * memory. Chains the descriptors as a singly-linked list; the descriptor's + * next * pointer specifies the bus address of the next descriptor. + * + * + * @dev Pointer to pci_dev + * @number Number of descriptors to be allocated + * @desc_bus_p Pointer where to store the first descriptor bus address + * + * @return Virtual address of the first descriptor + * + */ +static void transfer_desc_init(struct xdma_transfer *transfer, int count) +{ + struct xdma_desc *desc_virt = transfer->desc_virt; + dma_addr_t desc_bus = transfer->desc_bus; + int i; + int adj = count - 1; + int extra_adj; + u32 temp_control; + + BUG_ON(count > XDMA_TRANSFER_MAX_DESC); + + /* create singly-linked list for SG DMA controller */ + for (i = 0; i < count - 1; i++) { + /* increment bus address to next in array */ + desc_bus += sizeof(struct xdma_desc); + + /* singly-linked list uses bus addresses */ + desc_virt[i].next_lo = cpu_to_le32(PCI_DMA_L(desc_bus)); + desc_virt[i].next_hi = cpu_to_le32(PCI_DMA_H(desc_bus)); + desc_virt[i].bytes = cpu_to_le32(0); + + /* any adjacent descriptors? */ + if (adj > 0) { + extra_adj = adj - 1; + if (extra_adj > MAX_EXTRA_ADJ) + extra_adj = MAX_EXTRA_ADJ; + + adj--; + } else { + extra_adj = 0; + } + + temp_control = DESC_MAGIC | (extra_adj << 8); + + desc_virt[i].control = cpu_to_le32(temp_control); + } + /* { i = number - 1 } */ + /* zero the last descriptor next pointer */ + desc_virt[i].next_lo = cpu_to_le32(0); + desc_virt[i].next_hi = cpu_to_le32(0); + desc_virt[i].bytes = cpu_to_le32(0); + + temp_control = DESC_MAGIC; + + desc_virt[i].control = cpu_to_le32(temp_control); +} + +/* xdma_desc_link() - Link two descriptors + * + * Link the first descriptor to a second descriptor, or terminate the first. + * + * @first first descriptor + * @second second descriptor, or NULL if first descriptor must be set as last. + * @second_bus bus address of second descriptor + */ +static void xdma_desc_link(struct xdma_desc *first, struct xdma_desc *second, + dma_addr_t second_bus) +{ + /* + * remember reserved control in first descriptor, but zero + * extra_adjacent! + */ + /* RTO - what's this about? Shouldn't it be 0x0000c0ffUL? */ + u32 control = le32_to_cpu(first->control) & 0x0000f0ffUL; + /* second descriptor given? */ + if (second) { + /* + * link last descriptor of 1st array to first descriptor of + * 2nd array + */ + first->next_lo = cpu_to_le32(PCI_DMA_L(second_bus)); + first->next_hi = cpu_to_le32(PCI_DMA_H(second_bus)); + WARN_ON(first->next_hi); + /* no second descriptor given */ + } else { + /* first descriptor is the last */ + first->next_lo = 0; + first->next_hi = 0; + } + /* merge magic, extra_adjacent and control field */ + control |= DESC_MAGIC; + + /* write bytes and next_num */ + first->control = cpu_to_le32(control); +} + +/* xdma_desc_adjacent -- Set how many descriptors are adjacent to this one */ +static void xdma_desc_adjacent(struct xdma_desc *desc, int next_adjacent) +{ + int extra_adj = 0; + /* remember reserved and control bits */ + u32 control = le32_to_cpu(desc->control) & 0x0000f0ffUL; + u32 max_adj_4k = 0; + + if (next_adjacent > 0) { + extra_adj = next_adjacent - 1; + if (extra_adj > MAX_EXTRA_ADJ){ + extra_adj = MAX_EXTRA_ADJ; + } + max_adj_4k = (0x1000 - ((le32_to_cpu(desc->next_lo))&0xFFF))/32 - 1; + if (extra_adj>max_adj_4k) { + extra_adj = max_adj_4k; + } + if(extra_adj<0){ + printk("Warning: extra_adj<0, converting it to 0\n"); + extra_adj = 0; + } + } + /* merge adjacent and control field */ + control |= 0xAD4B0000UL | (extra_adj << 8); + /* write control and next_adjacent */ + desc->control = cpu_to_le32(control); +} + +/* xdma_desc_control -- Set complete control field of a descriptor. */ +static void xdma_desc_control_set(struct xdma_desc *first, u32 control_field) +{ + /* remember magic and adjacent number */ + u32 control = le32_to_cpu(first->control) & ~(LS_BYTE_MASK); + + BUG_ON(control_field & ~(LS_BYTE_MASK)); + /* merge adjacent and control field */ + control |= control_field; + /* write control and next_adjacent */ + first->control = cpu_to_le32(control); +} + +/* xdma_desc_clear -- Clear bits in control field of a descriptor. */ +static void xdma_desc_control_clear(struct xdma_desc *first, u32 clear_mask) +{ + /* remember magic and adjacent number */ + u32 control = le32_to_cpu(first->control); + + BUG_ON(clear_mask & ~(LS_BYTE_MASK)); + + /* merge adjacent and control field */ + control &= (~clear_mask); + /* write control and next_adjacent */ + first->control = cpu_to_le32(control); +} + +/* xdma_desc_done - recycle cache-coherent linked list of descriptors. + * + * @dev Pointer to pci_dev + * @number Number of descriptors to be allocated + * @desc_virt Pointer to (i.e. virtual address of) first descriptor in list + * @desc_bus Bus address of first descriptor in list + */ +static inline void xdma_desc_done(struct xdma_desc *desc_virt) +{ + memset(desc_virt, 0, XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc)); +} + +/* xdma_desc() - Fill a descriptor with the transfer details + * + * @desc pointer to descriptor to be filled + * @addr root complex address + * @ep_addr end point address + * @len number of bytes, must be a (non-negative) multiple of 4. + * @dir, dma direction + * is the end point address. If zero, vice versa. + * + * Does not modify the next pointer + */ +static void xdma_desc_set(struct xdma_desc *desc, dma_addr_t rc_bus_addr, + u64 ep_addr, int len, int dir) +{ + /* transfer length */ + desc->bytes = cpu_to_le32(len); + if (dir == DMA_TO_DEVICE) { + /* read from root complex memory (source address) */ + desc->src_addr_lo = cpu_to_le32(PCI_DMA_L(rc_bus_addr)); + desc->src_addr_hi = cpu_to_le32(PCI_DMA_H(rc_bus_addr)); + /* write to end point address (destination address) */ + desc->dst_addr_lo = cpu_to_le32(PCI_DMA_L(ep_addr)); + desc->dst_addr_hi = cpu_to_le32(PCI_DMA_H(ep_addr)); + } else { + /* read from end point address (source address) */ + desc->src_addr_lo = cpu_to_le32(PCI_DMA_L(ep_addr)); + desc->src_addr_hi = cpu_to_le32(PCI_DMA_H(ep_addr)); + /* write to root complex memory (destination address) */ + desc->dst_addr_lo = cpu_to_le32(PCI_DMA_L(rc_bus_addr)); + desc->dst_addr_hi = cpu_to_le32(PCI_DMA_H(rc_bus_addr)); + } +} + +/* + * should hold the engine->lock; + */ +static void transfer_abort(struct xdma_engine *engine, + struct xdma_transfer *transfer) +{ + struct xdma_transfer *head; + + BUG_ON(!engine); + BUG_ON(!transfer); + BUG_ON(transfer->desc_num == 0); + + pr_info("abort transfer 0x%p, desc %d, engine desc queued %d.\n", + transfer, transfer->desc_num, engine->desc_dequeued); + + head = list_entry(engine->transfer_list.next, struct xdma_transfer, + entry); + if (head == transfer) + list_del(engine->transfer_list.next); + else + pr_info("engine %s, transfer 0x%p NOT found, 0x%p.\n", + engine->name, transfer, head); + + if (transfer->state == TRANSFER_STATE_SUBMITTED) + transfer->state = TRANSFER_STATE_ABORTED; +} + +/* transfer_queue() - Queue a DMA transfer on the engine + * + * @engine DMA engine doing the transfer + * @transfer DMA transfer submitted to the engine + * + * Takes and releases the engine spinlock + */ +static int transfer_queue(struct xdma_engine *engine, + struct xdma_transfer *transfer) +{ + int rv = 0; + struct xdma_transfer *transfer_started; + struct xdma_dev *xdev; + unsigned long flags; + + BUG_ON(!engine); + BUG_ON(!engine->xdev); + BUG_ON(!transfer); + BUG_ON(transfer->desc_num == 0); + dbg_tfr("transfer_queue(transfer=0x%p).\n", transfer); + + xdev = engine->xdev; + if (xdma_device_flag_check(xdev, XDEV_FLAG_OFFLINE)) { + pr_info("dev 0x%p offline, transfer 0x%p not queued.\n", + xdev, transfer); + return -EBUSY; + } + + /* lock the engine state */ + spin_lock_irqsave(&engine->lock, flags); + + engine->prev_cpu = get_cpu(); + put_cpu(); + + /* engine is being shutdown; do not accept new transfers */ + if (engine->shutdown & ENGINE_SHUTDOWN_REQUEST) { + pr_info("engine %s offline, transfer 0x%p not queued.\n", + engine->name, transfer); + rv = -EBUSY; + goto shutdown; + } + + /* mark the transfer as submitted */ + transfer->state = TRANSFER_STATE_SUBMITTED; + /* add transfer to the tail of the engine transfer queue */ + list_add_tail(&transfer->entry, &engine->transfer_list); + + /* engine is idle? */ + if (!engine->running) { + /* start engine */ + dbg_tfr("transfer_queue(): starting %s engine.\n", + engine->name); + transfer_started = engine_start(engine); + dbg_tfr("transfer=0x%p started %s engine with transfer 0x%p.\n", + transfer, engine->name, transfer_started); + } else { + dbg_tfr("transfer=0x%p queued, with %s engine running.\n", + transfer, engine->name); + } + +shutdown: + /* unlock the engine state */ + dbg_tfr("engine->running = %d\n", engine->running); + spin_unlock_irqrestore(&engine->lock, flags); + return rv; +} + +static void engine_alignments(struct xdma_engine *engine) +{ + u32 w; + u32 align_bytes; + u32 granularity_bytes; + u32 address_bits; + + w = read_register(&engine->regs->alignments); + dbg_init("engine %p name %s alignments=0x%08x\n", engine, + engine->name, (int)w); + + /* RTO - add some macros to extract these fields */ + align_bytes = (w & 0x00ff0000U) >> 16; + granularity_bytes = (w & 0x0000ff00U) >> 8; + address_bits = (w & 0x000000ffU); + + dbg_init("align_bytes = %d\n", align_bytes); + dbg_init("granularity_bytes = %d\n", granularity_bytes); + dbg_init("address_bits = %d\n", address_bits); + + if (w) { + engine->addr_align = align_bytes; + engine->len_granularity = granularity_bytes; + engine->addr_bits = address_bits; + } else { + /* Some default values if alignments are unspecified */ + engine->addr_align = 1; + engine->len_granularity = 1; + engine->addr_bits = 64; + } +} + +static void engine_free_resource(struct xdma_engine *engine) +{ + struct xdma_dev *xdev = engine->xdev; + + /* Release memory use for descriptor writebacks */ + if (engine->poll_mode_addr_virt) { + dbg_sg("Releasing memory for descriptor writeback\n"); + dma_free_coherent(&xdev->pdev->dev, + sizeof(struct xdma_poll_wb), + engine->poll_mode_addr_virt, + engine->poll_mode_bus); + dbg_sg("Released memory for descriptor writeback\n"); + engine->poll_mode_addr_virt = NULL; + } + + if (engine->desc) { + dbg_init("device %s, engine %s pre-alloc desc 0x%p,0x%llx.\n", + dev_name(&xdev->pdev->dev), engine->name, + engine->desc, engine->desc_bus); + dma_free_coherent(&xdev->pdev->dev, + XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc), + engine->desc, engine->desc_bus); + engine->desc = NULL; + } + + if (engine->cyclic_result) { + dma_free_coherent(&xdev->pdev->dev, + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result), + engine->cyclic_result, engine->cyclic_result_bus); + engine->cyclic_result = NULL; + } +} + +static void engine_destroy(struct xdma_dev *xdev, struct xdma_engine *engine) +{ + BUG_ON(!xdev); + BUG_ON(!engine); + + dbg_sg("Shutting down engine %s%d", engine->name, engine->channel); + + /* Disable interrupts to stop processing new events during shutdown */ + write_register(0x0, &engine->regs->interrupt_enable_mask, + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); + + if (enable_credit_mp && engine->streaming && + engine->dir == DMA_FROM_DEVICE) { + u32 reg_value = (0x1 << engine->channel) << 16; + struct sgdma_common_regs *reg = (struct sgdma_common_regs *) + (xdev->bar[xdev->config_bar_idx] + + (0x6*TARGET_SPACING)); + write_register(reg_value, ®->credit_mode_enable_w1c, 0); + } + + /* Release memory use for descriptor writebacks */ + engine_free_resource(engine); + + memset(engine, 0, sizeof(struct xdma_engine)); + /* Decrement the number of engines available */ + xdev->engines_num--; +} + +/** + *engine_cyclic_stop() - stop a cyclic transfer running on an SG DMA engine + * + *engine->lock must be taken + */ +struct xdma_transfer *engine_cyclic_stop(struct xdma_engine *engine) +{ + struct xdma_transfer *transfer = 0; + + /* transfers on queue? */ + if (!list_empty(&engine->transfer_list)) { + /* pick first transfer on the queue (was submitted to engine) */ + transfer = list_entry(engine->transfer_list.next, + struct xdma_transfer, entry); + BUG_ON(!transfer); + + xdma_engine_stop(engine); + + if (transfer->cyclic) { + if (engine->xdma_perf) + dbg_perf("Stopping perf transfer on %s\n", + engine->name); + else + dbg_perf("Stopping cyclic transfer on %s\n", + engine->name); + /* make sure the handler sees correct transfer state */ + transfer->cyclic = 1; + /* + * set STOP flag and interrupt on completion, on the + * last descriptor + */ + xdma_desc_control_set( + transfer->desc_virt + transfer->desc_num - 1, + XDMA_DESC_COMPLETED | XDMA_DESC_STOPPED); + } else { + dbg_sg("(engine=%p) running transfer is not cyclic\n", + engine); + } + } else { + dbg_sg("(engine=%p) found not running transfer.\n", engine); + } + return transfer; +} +EXPORT_SYMBOL_GPL(engine_cyclic_stop); + +static int engine_writeback_setup(struct xdma_engine *engine) +{ + u32 w; + struct xdma_dev *xdev; + struct xdma_poll_wb *writeback; + + BUG_ON(!engine); + xdev = engine->xdev; + BUG_ON(!xdev); + + /* + * RTO - doing the allocation per engine is wasteful since a full page + * is allocated each time - better to allocate one page for the whole + * device during probe() and set per-engine offsets here + */ + writeback = (struct xdma_poll_wb *)engine->poll_mode_addr_virt; + writeback->completed_desc_count = 0; + + dbg_init("Setting writeback location to 0x%llx for engine %p", + engine->poll_mode_bus, engine); + w = cpu_to_le32(PCI_DMA_L(engine->poll_mode_bus)); + write_register(w, &engine->regs->poll_mode_wb_lo, + (unsigned long)(&engine->regs->poll_mode_wb_lo) - + (unsigned long)(&engine->regs)); + w = cpu_to_le32(PCI_DMA_H(engine->poll_mode_bus)); + write_register(w, &engine->regs->poll_mode_wb_hi, + (unsigned long)(&engine->regs->poll_mode_wb_hi) - + (unsigned long)(&engine->regs)); + + return 0; +} + + +/* engine_create() - Create an SG DMA engine bookkeeping data structure + * + * An SG DMA engine consists of the resources for a single-direction transfer + * queue; the SG DMA hardware, the software queue and interrupt handling. + * + * @dev Pointer to pci_dev + * @offset byte address offset in BAR[xdev->config_bar_idx] resource for the + * SG DMA * controller registers. + * @dir: DMA_TO/FROM_DEVICE + * @streaming Whether the engine is attached to AXI ST (rather than MM) + */ +static int engine_init_regs(struct xdma_engine *engine) +{ + u32 reg_value; + int rv = 0; + + write_register(XDMA_CTRL_NON_INCR_ADDR, &engine->regs->control_w1c, + (unsigned long)(&engine->regs->control_w1c) - + (unsigned long)(&engine->regs)); + + engine_alignments(engine); + + /* Configure error interrupts by default */ + reg_value = XDMA_CTRL_IE_DESC_ALIGN_MISMATCH; + reg_value |= XDMA_CTRL_IE_MAGIC_STOPPED; + reg_value |= XDMA_CTRL_IE_MAGIC_STOPPED; + reg_value |= XDMA_CTRL_IE_READ_ERROR; + reg_value |= XDMA_CTRL_IE_DESC_ERROR; + + /* if using polled mode, configure writeback address */ + if (poll_mode) { + rv = engine_writeback_setup(engine); + if (rv) { + dbg_init("%s descr writeback setup failed.\n", + engine->name); + goto fail_wb; + } + } else { + /* enable the relevant completion interrupts */ + reg_value |= XDMA_CTRL_IE_DESC_STOPPED; + reg_value |= XDMA_CTRL_IE_DESC_COMPLETED; + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) + reg_value |= XDMA_CTRL_IE_IDLE_STOPPED; + } + + /* Apply engine configurations */ + write_register(reg_value, &engine->regs->interrupt_enable_mask, + (unsigned long)(&engine->regs->interrupt_enable_mask) - + (unsigned long)(&engine->regs)); + + engine->interrupt_enable_mask_value = reg_value; + + /* only enable credit mode for AXI-ST C2H */ + if (enable_credit_mp && engine->streaming && + engine->dir == DMA_FROM_DEVICE) { + + struct xdma_dev *xdev = engine->xdev; + u32 reg_value = (0x1 << engine->channel) << 16; + struct sgdma_common_regs *reg = (struct sgdma_common_regs *) + (xdev->bar[xdev->config_bar_idx] + + (0x6*TARGET_SPACING)); + + write_register(reg_value, ®->credit_mode_enable_w1s, 0); + } + + return 0; + +fail_wb: + return rv; +} + +static int engine_alloc_resource(struct xdma_engine *engine) +{ + struct xdma_dev *xdev = engine->xdev; + + engine->desc = dma_alloc_coherent(&xdev->pdev->dev, + XDMA_TRANSFER_MAX_DESC * sizeof(struct xdma_desc), + &engine->desc_bus, GFP_KERNEL); + if (!engine->desc) { + pr_warn("dev %s, %s pre-alloc desc OOM.\n", + dev_name(&xdev->pdev->dev), engine->name); + goto err_out; + } + + if (poll_mode) { + engine->poll_mode_addr_virt = dma_alloc_coherent( + &xdev->pdev->dev, + sizeof(struct xdma_poll_wb), + &engine->poll_mode_bus, GFP_KERNEL); + if (!engine->poll_mode_addr_virt) { + pr_warn("%s, %s poll pre-alloc writeback OOM.\n", + dev_name(&xdev->pdev->dev), engine->name); + goto err_out; + } + } + + if (engine->streaming && engine->dir == DMA_FROM_DEVICE) { + engine->cyclic_result = dma_alloc_coherent(&xdev->pdev->dev, + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result), + &engine->cyclic_result_bus, GFP_KERNEL); + + if (!engine->cyclic_result) { + pr_warn("%s, %s pre-alloc result OOM.\n", + dev_name(&xdev->pdev->dev), engine->name); + goto err_out; + } + } + + return 0; + +err_out: + engine_free_resource(engine); + return -ENOMEM; +} + +static int engine_init(struct xdma_engine *engine, struct xdma_dev *xdev, + int offset, enum dma_data_direction dir, int channel) +{ + int rv; + u32 val; + + dbg_init("channel %d, offset 0x%x, dir %d.\n", channel, offset, dir); + + /* set magic */ + engine->magic = MAGIC_ENGINE; + + engine->channel = channel; + + /* engine interrupt request bit */ + engine->irq_bitmask = (1 << XDMA_ENG_IRQ_NUM) - 1; + engine->irq_bitmask <<= (xdev->engines_num * XDMA_ENG_IRQ_NUM); + engine->bypass_offset = xdev->engines_num * BYPASS_MODE_SPACING; + + /* parent */ + engine->xdev = xdev; + /* register address */ + engine->regs = (xdev->bar[xdev->config_bar_idx] + offset); + engine->sgdma_regs = xdev->bar[xdev->config_bar_idx] + offset + + SGDMA_OFFSET_FROM_CHANNEL; + val = read_register(&engine->regs->identifier); + if (val & 0x8000U) + engine->streaming = 1; + + /* remember SG DMA direction */ + engine->dir = dir; + sprintf(engine->name, "%d-%s%d-%s", xdev->idx, + (dir == DMA_TO_DEVICE) ? "H2C" : "C2H", channel, + engine->streaming ? "ST" : "MM"); + + dbg_init("engine %p name %s irq_bitmask=0x%08x\n", engine, engine->name, + (int)engine->irq_bitmask); + + /* initialize the deferred work for transfer completion */ + INIT_WORK(&engine->work, engine_service_work); + + if (dir == DMA_TO_DEVICE) + xdev->mask_irq_h2c |= engine->irq_bitmask; + else + xdev->mask_irq_c2h |= engine->irq_bitmask; + xdev->engines_num++; + + rv = engine_alloc_resource(engine); + if (rv) + return rv; + + rv = engine_init_regs(engine); + if (rv) + return rv; + + return 0; +} + +/* transfer_destroy() - free transfer */ +static void transfer_destroy(struct xdma_dev *xdev, struct xdma_transfer *xfer) +{ + /* free descriptors */ + xdma_desc_done(xfer->desc_virt); + + if (xfer->last_in_request && (xfer->flags & XFER_FLAG_NEED_UNMAP)) { + struct sg_table *sgt = xfer->sgt; + + if (sgt->nents) { + pci_unmap_sg(xdev->pdev, sgt->sgl, sgt->nents, + xfer->dir); + sgt->nents = 0; + } + } +} + +static int transfer_build(struct xdma_engine *engine, + struct xdma_request_cb *req, unsigned int desc_max) +{ + struct xdma_transfer *xfer = &req->xfer; + struct sw_desc *sdesc = &(req->sdesc[req->sw_desc_idx]); + int i = 0; + int j = 0; + + for (; i < desc_max; i++, j++, sdesc++) { + dbg_desc("sw desc %d/%u: 0x%llx, 0x%x, ep 0x%llx.\n", + i + req->sw_desc_idx, req->sw_desc_cnt, + sdesc->addr, sdesc->len, req->ep_addr); + + /* fill in descriptor entry j with transfer details */ + xdma_desc_set(xfer->desc_virt + j, sdesc->addr, req->ep_addr, + sdesc->len, xfer->dir); + xfer->len += sdesc->len; + + /* for non-inc-add mode don't increment ep_addr */ + if (!engine->non_incr_addr) + req->ep_addr += sdesc->len; + } + req->sw_desc_idx += desc_max; + return 0; +} + +static int transfer_init(struct xdma_engine *engine, struct xdma_request_cb *req) +{ + struct xdma_transfer *xfer = &req->xfer; + unsigned int desc_max = min_t(unsigned int, + req->sw_desc_cnt - req->sw_desc_idx, + XDMA_TRANSFER_MAX_DESC); + int i = 0; + int last = 0; + u32 control; + + memset(xfer, 0, sizeof(*xfer)); + + /* initialize wait queue */ + init_waitqueue_head(&xfer->wq); + + /* remember direction of transfer */ + xfer->dir = engine->dir; + + xfer->desc_virt = engine->desc; + xfer->desc_bus = engine->desc_bus; + + transfer_desc_init(xfer, desc_max); + + dbg_sg("transfer->desc_bus = 0x%llx.\n", (u64)xfer->desc_bus); + + transfer_build(engine, req, desc_max); + + /* terminate last descriptor */ + last = desc_max - 1; + xdma_desc_link(xfer->desc_virt + last, 0, 0); + /* stop engine, EOP for AXI ST, req IRQ on last descriptor */ + control = XDMA_DESC_STOPPED; + control |= XDMA_DESC_EOP; + control |= XDMA_DESC_COMPLETED; + xdma_desc_control_set(xfer->desc_virt + last, control); + + xfer->desc_num = xfer->desc_adjacent = desc_max; + + dbg_sg("transfer 0x%p has %d descriptors\n", xfer, xfer->desc_num); + /* fill in adjacent numbers */ + for (i = 0; i < xfer->desc_num; i++) + xdma_desc_adjacent(xfer->desc_virt + i, xfer->desc_num - i - 1); + + return 0; +} + +#ifdef __LIBXDMA_DEBUG__ +static void sgt_dump(struct sg_table *sgt) +{ + int i; + struct scatterlist *sg = sgt->sgl; + + pr_info("sgt 0x%p, sgl 0x%p, nents %u/%u.\n", + sgt, sgt->sgl, sgt->nents, sgt->orig_nents); + + for (i = 0; i < sgt->orig_nents; i++, sg = sg_next(sg)) + pr_info("%d, 0x%p, pg 0x%p,%u+%u, dma 0x%llx,%u.\n", + i, sg, sg_page(sg), sg->offset, sg->length, + sg_dma_address(sg), sg_dma_len(sg)); +} + +static void xdma_request_cb_dump(struct xdma_request_cb *req) +{ + int i; + + pr_info("request 0x%p, total %u, ep 0x%llx, sw_desc %u, sgt 0x%p.\n", + req, req->total_len, req->ep_addr, req->sw_desc_cnt, req->sgt); + sgt_dump(req->sgt); + for (i = 0; i < req->sw_desc_cnt; i++) + pr_info("%d/%u, 0x%llx, %u.\n", + i, req->sw_desc_cnt, req->sdesc[i].addr, + req->sdesc[i].len); +} +#endif + +static void xdma_request_free(struct xdma_request_cb *req) +{ + if (((unsigned long)req) >= VMALLOC_START && + ((unsigned long)req) < VMALLOC_END) + vfree(req); + else + kfree(req); +} + +static struct xdma_request_cb * xdma_request_alloc(unsigned int sdesc_nr) +{ + struct xdma_request_cb *req; + unsigned int size = sizeof(struct xdma_request_cb) + + sdesc_nr * sizeof(struct sw_desc); + + req = kzalloc(size, GFP_KERNEL); + if (!req) { + req = vmalloc(size); + if (req) + memset(req, 0, size); + } + if (!req) { + pr_info("OOM, %u sw_desc, %u.\n", sdesc_nr, size); + return NULL; + } + + return req; +} + +static struct xdma_request_cb * xdma_init_request(struct sg_table *sgt, + u64 ep_addr) +{ + struct xdma_request_cb *req; + struct scatterlist *sg = sgt->sgl; + int max = sgt->nents; + int extra = 0; + int i, j = 0; + + for (i = 0; i < max; i++, sg = sg_next(sg)) { + unsigned int len = sg_dma_len(sg); + + if (unlikely(len > desc_blen_max)) + extra += (len + desc_blen_max - 1) / desc_blen_max; + } + +//pr_info("ep 0x%llx, desc %u+%u.\n", ep_addr, max, extra); + + max += extra; + req = xdma_request_alloc(max); + if (!req) + return NULL; + + req->sgt = sgt; + req->ep_addr = ep_addr; + + for (i = 0, sg = sgt->sgl; i < sgt->nents; i++, sg = sg_next(sg)) { + unsigned int tlen = sg_dma_len(sg); + dma_addr_t addr = sg_dma_address(sg); + + req->total_len += tlen; + while (tlen) { + req->sdesc[j].addr = addr; + if (tlen > desc_blen_max) { + req->sdesc[j].len = desc_blen_max; + addr += desc_blen_max; + tlen -= desc_blen_max; + } else { + req->sdesc[j].len = tlen; + tlen = 0; + } + j++; + } + } + BUG_ON(j > max); + + req->sw_desc_cnt = j; +#ifdef __LIBXDMA_DEBUG__ + xdma_request_cb_dump(req); +#endif + return req; +} + +ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr, + struct sg_table *sgt, bool dma_mapped, int timeout_ms) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + struct xdma_engine *engine; + int rv = 0; + ssize_t done = 0; + struct scatterlist *sg = sgt->sgl; + int nents; + enum dma_data_direction dir = write ? DMA_TO_DEVICE : DMA_FROM_DEVICE; + struct xdma_request_cb *req = NULL; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + if (write == 1) { + if (channel >= xdev->h2c_channel_max) { + pr_warn("H2C channel %d >= %d.\n", + channel, xdev->h2c_channel_max); + return -EINVAL; + } + engine = &xdev->engine_h2c[channel]; + } else if (write == 0) { + if (channel >= xdev->c2h_channel_max) { + pr_warn("C2H channel %d >= %d.\n", + channel, xdev->c2h_channel_max); + return -EINVAL; + } + engine = &xdev->engine_c2h[channel]; + } else { + pr_warn("write %d, exp. 0|1.\n", write); + return -EINVAL; + } + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + xdev = engine->xdev; + if (xdma_device_flag_check(xdev, XDEV_FLAG_OFFLINE)) { + pr_info("xdev 0x%p, offline.\n", xdev); + return -EBUSY; + } + + /* check the direction */ + if (engine->dir != dir) { + pr_info("0x%p, %s, %d, W %d, 0x%x/0x%x mismatch.\n", + engine, engine->name, channel, write, engine->dir, dir); + return -EINVAL; + } + + if (!dma_mapped) { + nents = pci_map_sg(xdev->pdev, sg, sgt->orig_nents, dir); + if (!nents) { + pr_info("map sgl failed, sgt 0x%p.\n", sgt); + return -EIO; + } + sgt->nents = nents; + } else { + BUG_ON(!sgt->nents); + } + + req = xdma_init_request(sgt, ep_addr); + if (!req) { + rv = -ENOMEM; + goto unmap_sgl; + } + + dbg_tfr("%s, len %u sg cnt %u.\n", + engine->name, req->total_len, req->sw_desc_cnt); + + sg = sgt->sgl; + nents = req->sw_desc_cnt; + while (nents) { + unsigned long flags; + struct xdma_transfer *xfer; + + /* one transfer at a time */ + spin_lock(&engine->desc_lock); + + /* build transfer */ + rv = transfer_init(engine, req); + if (rv < 0) { + spin_unlock(&engine->desc_lock); + goto unmap_sgl; + } + xfer = &req->xfer; + + if (!dma_mapped) + xfer->flags = XFER_FLAG_NEED_UNMAP; + + /* last transfer for the given request? */ + nents -= xfer->desc_num; + if (!nents) { + xfer->last_in_request = 1; + xfer->sgt = sgt; + } + + dbg_tfr("xfer, %u, ep 0x%llx, done %lu, sg %u/%u.\n", + xfer->len, req->ep_addr, done, req->sw_desc_idx, + req->sw_desc_cnt); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); +#endif + + rv = transfer_queue(engine, xfer); + if (rv < 0) { + spin_unlock(&engine->desc_lock); + pr_info("unable to submit %s, %d.\n", engine->name, rv); + goto unmap_sgl; + } + + /* + * When polling, determine how many descriptors have been queued * on the engine to determine the writeback value expected + */ + if (poll_mode) { + unsigned int desc_count; + + spin_lock_irqsave(&engine->lock, flags); + desc_count = xfer->desc_num; + spin_unlock_irqrestore(&engine->lock, flags); + + dbg_tfr("%s poll desc_count=%d\n", + engine->name, desc_count); + rv = engine_service_poll(engine, desc_count); + + } else { + rv = wait_event_interruptible_timeout(xfer->wq, + (xfer->state != TRANSFER_STATE_SUBMITTED), + msecs_to_jiffies(timeout_ms)); + } + + spin_lock_irqsave(&engine->lock, flags); + + switch(xfer->state) { + case TRANSFER_STATE_COMPLETED: + spin_unlock_irqrestore(&engine->lock, flags); + + dbg_tfr("transfer %p, %u, ep 0x%llx compl, +%lu.\n", + xfer, xfer->len, req->ep_addr - xfer->len, done); + done += xfer->len; + rv = 0; + break; + case TRANSFER_STATE_FAILED: + pr_info("xfer 0x%p,%u, failed, ep 0x%llx.\n", + xfer, xfer->len, req->ep_addr - xfer->len); + spin_unlock_irqrestore(&engine->lock, flags); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); + sgt_dump(sgt); +#endif + rv = -EIO; + break; + default: + /* transfer can still be in-flight */ + pr_info("xfer 0x%p,%u, s 0x%x timed out, ep 0x%llx.\n", + xfer, xfer->len, xfer->state, req->ep_addr); + engine_status_read(engine, 0, 1); + //engine_status_dump(engine); + transfer_abort(engine, xfer); + + xdma_engine_stop(engine); + spin_unlock_irqrestore(&engine->lock, flags); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); + sgt_dump(sgt); +#endif + rv = -ERESTARTSYS; + break; + } + + transfer_destroy(xdev, xfer); + spin_unlock(&engine->desc_lock); + + if (rv < 0) + goto unmap_sgl; + } /* while (sg) */ + +unmap_sgl: + if (!dma_mapped && sgt->nents) { + pci_unmap_sg(xdev->pdev, sgt->sgl, sgt->orig_nents, dir); + sgt->nents = 0; + } + + if (req) + xdma_request_free(req); + + if (rv < 0) + return rv; + + return done; +} +EXPORT_SYMBOL_GPL(xdma_xfer_submit); + +int xdma_performance_submit(struct xdma_dev *xdev, struct xdma_engine *engine) +{ + u8 *buffer_virt; + u32 max_consistent_size = 128 * 32 * 1024; /* 1024 pages, 4MB */ + dma_addr_t buffer_bus; /* bus address */ + struct xdma_transfer *transfer; + u64 ep_addr = 0; + int num_desc_in_a_loop = 128; + int size_in_desc = engine->xdma_perf->transfer_size; + int size = size_in_desc * num_desc_in_a_loop; + int i; + + BUG_ON(size_in_desc > max_consistent_size); + + if (size > max_consistent_size) { + size = max_consistent_size; + num_desc_in_a_loop = size / size_in_desc; + } + + buffer_virt = dma_alloc_coherent(&xdev->pdev->dev, size, + &buffer_bus, GFP_KERNEL); + + /* allocate transfer data structure */ + transfer = kzalloc(sizeof(struct xdma_transfer), GFP_KERNEL); + BUG_ON(!transfer); + + /* 0 = write engine (to_dev=0) , 1 = read engine (to_dev=1) */ + transfer->dir = engine->dir; + /* set number of descriptors */ + transfer->desc_num = num_desc_in_a_loop; + + /* allocate descriptor list */ + if (!engine->desc) { + engine->desc = dma_alloc_coherent(&xdev->pdev->dev, + num_desc_in_a_loop * sizeof(struct xdma_desc), + &engine->desc_bus, GFP_KERNEL); + BUG_ON(!engine->desc); + dbg_init("device %s, engine %s pre-alloc desc 0x%p,0x%llx.\n", + dev_name(&xdev->pdev->dev), engine->name, + engine->desc, engine->desc_bus); + } + transfer->desc_virt = engine->desc; + transfer->desc_bus = engine->desc_bus; + + transfer_desc_init(transfer, transfer->desc_num); + + dbg_sg("transfer->desc_bus = 0x%llx.\n", (u64)transfer->desc_bus); + + for (i = 0; i < transfer->desc_num; i++) { + struct xdma_desc *desc = transfer->desc_virt + i; + dma_addr_t rc_bus_addr = buffer_bus + size_in_desc * i; + + /* fill in descriptor entry with transfer details */ + xdma_desc_set(desc, rc_bus_addr, ep_addr, size_in_desc, + engine->dir); + } + + /* stop engine and request interrupt on last descriptor */ + xdma_desc_control_set(transfer->desc_virt, 0); + /* create a linked loop */ + xdma_desc_link(transfer->desc_virt + transfer->desc_num - 1, + transfer->desc_virt, transfer->desc_bus); + + transfer->cyclic = 1; + + /* initialize wait queue */ + init_waitqueue_head(&transfer->wq); + + //printk("=== Descriptor print for PERF \n"); + //transfer_dump(transfer); + + dbg_perf("Queueing XDMA I/O %s request for performance measurement.\n", + engine->dir ? "write (to dev)" : "read (from dev)"); + transfer_queue(engine, transfer); + return 0; + +} +EXPORT_SYMBOL_GPL(xdma_performance_submit); + +static struct xdma_dev *alloc_dev_instance(struct pci_dev *pdev) +{ + int i; + struct xdma_dev *xdev; + struct xdma_engine *engine; + + BUG_ON(!pdev); + + /* allocate zeroed device book keeping structure */ + xdev = kzalloc(sizeof(struct xdma_dev), GFP_KERNEL); + if (!xdev) { + pr_info("OOM, xdma_dev.\n"); + return NULL; + } + spin_lock_init(&xdev->lock); + + xdev->magic = MAGIC_DEVICE; + xdev->config_bar_idx = -1; + xdev->user_bar_idx = -1; + xdev->bypass_bar_idx = -1; + xdev->irq_line = -1; + + /* create a driver to device reference */ + xdev->pdev = pdev; + dbg_init("xdev = 0x%p\n", xdev); + + /* Set up data user IRQ data structures */ + for (i = 0; i < 16; i++) { + xdev->user_irq[i].xdev = xdev; + spin_lock_init(&xdev->user_irq[i].events_lock); + init_waitqueue_head(&xdev->user_irq[i].events_wq); + xdev->user_irq[i].handler = NULL; + xdev->user_irq[i].user_idx = i; /* 0 based */ + } + + engine = xdev->engine_h2c; + for (i = 0; i < XDMA_CHANNEL_NUM_MAX; i++, engine++) { + spin_lock_init(&engine->lock); + spin_lock_init(&engine->desc_lock); + INIT_LIST_HEAD(&engine->transfer_list); + init_waitqueue_head(&engine->shutdown_wq); + init_waitqueue_head(&engine->xdma_perf_wq); + } + + engine = xdev->engine_c2h; + for (i = 0; i < XDMA_CHANNEL_NUM_MAX; i++, engine++) { + spin_lock_init(&engine->lock); + spin_lock_init(&engine->desc_lock); + INIT_LIST_HEAD(&engine->transfer_list); + init_waitqueue_head(&engine->shutdown_wq); + init_waitqueue_head(&engine->xdma_perf_wq); + } + + return xdev; +} + +static int request_regions(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + int rv; + + BUG_ON(!xdev); + BUG_ON(!pdev); + + dbg_init("pci_request_regions()\n"); + rv = pci_request_regions(pdev, xdev->mod_name); + /* could not request all regions? */ + if (rv) { + dbg_init("pci_request_regions() = %d, device in use?\n", rv); + /* assume device is in use so do not disable it later */ + xdev->regions_in_use = 1; + } else { + xdev->got_regions = 1; + } + + return rv; +} + +static int set_dma_mask(struct pci_dev *pdev) +{ + BUG_ON(!pdev); + + dbg_init("sizeof(dma_addr_t) == %ld\n", sizeof(dma_addr_t)); + /* 64-bit addressing capability for XDMA? */ + if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + /* query for DMA transfer */ + /* @see Documentation/DMA-mapping.txt */ + dbg_init("pci_set_dma_mask()\n"); + /* use 64-bit DMA */ + dbg_init("Using a 64-bit DMA mask.\n"); + /* use 32-bit DMA for descriptors */ + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + /* use 64-bit DMA, 32-bit for consistent */ + } else if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) { + dbg_init("Could not set 64-bit DMA mask.\n"); + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + /* use 32-bit DMA */ + dbg_init("Using a 32-bit DMA mask.\n"); + } else { + dbg_init("No suitable DMA possible.\n"); + return -EINVAL; + } + + return 0; +} + +static u32 get_engine_channel_id(struct engine_regs *regs) +{ + u32 value; + + BUG_ON(!regs); + + value = read_register(®s->identifier); + + return (value & 0x00000f00U) >> 8; +} + +static u32 get_engine_id(struct engine_regs *regs) +{ + u32 value; + + BUG_ON(!regs); + + value = read_register(®s->identifier); + return (value & 0xffff0000U) >> 16; +} + +static void remove_engines(struct xdma_dev *xdev) +{ + struct xdma_engine *engine; + int i; + + BUG_ON(!xdev); + + /* iterate over channels */ + for (i = 0; i < xdev->h2c_channel_max; i++) { + engine = &xdev->engine_h2c[i]; + if (engine->magic == MAGIC_ENGINE) { + dbg_sg("Remove %s, %d", engine->name, i); + engine_destroy(xdev, engine); + dbg_sg("%s, %d removed", engine->name, i); + } + } + + for (i = 0; i < xdev->c2h_channel_max; i++) { + engine = &xdev->engine_c2h[i]; + if (engine->magic == MAGIC_ENGINE) { + dbg_sg("Remove %s, %d", engine->name, i); + engine_destroy(xdev, engine); + dbg_sg("%s, %d removed", engine->name, i); + } + } +} + +static int probe_for_engine(struct xdma_dev *xdev, enum dma_data_direction dir, + int channel) +{ + struct engine_regs *regs; + int offset = channel * CHANNEL_SPACING; + u32 engine_id; + u32 engine_id_expected; + u32 channel_id; + struct xdma_engine *engine; + int rv; + + /* register offset for the engine */ + /* read channels at 0x0000, write channels at 0x1000, + * channels at 0x100 interval */ + if (dir == DMA_TO_DEVICE) { + engine_id_expected = XDMA_ID_H2C; + engine = &xdev->engine_h2c[channel]; + } else { + offset += H2C_CHANNEL_OFFSET; + engine_id_expected = XDMA_ID_C2H; + engine = &xdev->engine_c2h[channel]; + } + + regs = xdev->bar[xdev->config_bar_idx] + offset; + engine_id = get_engine_id(regs); + channel_id = get_engine_channel_id(regs); + + if ((engine_id != engine_id_expected) || (channel_id != channel)) { + dbg_init("%s %d engine, reg off 0x%x, id mismatch 0x%x,0x%x," + "exp 0x%x,0x%x, SKIP.\n", + dir == DMA_TO_DEVICE ? "H2C" : "C2H", + channel, offset, engine_id, channel_id, + engine_id_expected, channel_id != channel); + return -EINVAL; + } + + dbg_init("found AXI %s %d engine, reg. off 0x%x, id 0x%x,0x%x.\n", + dir == DMA_TO_DEVICE ? "H2C" : "C2H", channel, + offset, engine_id, channel_id); + + /* allocate and initialize engine */ + rv = engine_init(engine, xdev, offset, dir, channel); + if (rv != 0) { + pr_info("failed to create AXI %s %d engine.\n", + dir == DMA_TO_DEVICE ? "H2C" : "C2H", + channel); + return rv; + } + + return 0; +} + +static int probe_engines(struct xdma_dev *xdev) +{ + int i; + int rv = 0; + + BUG_ON(!xdev); + + /* iterate over channels */ + for (i = 0; i < xdev->h2c_channel_max; i++) { + rv = probe_for_engine(xdev, DMA_TO_DEVICE, i); + if (rv) + break; + } + xdev->h2c_channel_max = i; + + for (i = 0; i < xdev->c2h_channel_max; i++) { + rv = probe_for_engine(xdev, DMA_FROM_DEVICE, i); + if (rv) + break; + } + xdev->c2h_channel_max = i; + + return 0; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0) +static void pci_enable_relaxed_ordering(struct pci_dev *pdev) +{ + pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN); +} +#else +static void pci_enable_relaxed_ordering(struct pci_dev *pdev) +{ + u16 v; + int pos; + + pos = pci_pcie_cap(pdev); + if (pos > 0) { + pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &v); + v |= PCI_EXP_DEVCTL_RELAX_EN; + pci_write_config_word(pdev, pos + PCI_EXP_DEVCTL, v); + } +} +#endif + +static void pci_check_extended_tag(struct xdma_dev *xdev, struct pci_dev *pdev) +{ + u16 cap; + u32 v; + void *__iomem reg; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,5,0) + pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &cap); +#else + int pos; + + pos = pci_pcie_cap(pdev); + if (pos > 0) + pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &cap); + else { + pr_info("pdev 0x%p, unable to access pcie cap.\n", pdev); + return; + } +#endif + + if ((cap & PCI_EXP_DEVCTL_EXT_TAG)) + return; + + /* extended tag not enabled */ + pr_info("0x%p EXT_TAG disabled.\n", pdev); + + if (xdev->config_bar_idx < 0) { + pr_info("pdev 0x%p, xdev 0x%p, config bar UNKNOWN.\n", + pdev, xdev); + return; + } + + reg = xdev->bar[xdev->config_bar_idx] + XDMA_OFS_CONFIG + 0x4C; + v = read_register(reg); + v = (v & 0xFF) | (((u32)32) << 8); + write_register(v, reg, XDMA_OFS_CONFIG + 0x4C); +} + +void *xdma_device_open(const char *mname, struct pci_dev *pdev, int *user_max, + int *h2c_channel_max, int *c2h_channel_max) +{ + struct xdma_dev *xdev = NULL; + int rv = 0; + + pr_info("%s device %s, 0x%p.\n", mname, dev_name(&pdev->dev), pdev); + + /* allocate zeroed device book keeping structure */ + xdev = alloc_dev_instance(pdev); + if (!xdev) + return NULL; + xdev->mod_name = mname; + xdev->user_max = *user_max; + xdev->h2c_channel_max = *h2c_channel_max; + xdev->c2h_channel_max = *c2h_channel_max; + + xdma_device_flag_set(xdev, XDEV_FLAG_OFFLINE); + xdev_list_add(xdev); + + if (xdev->user_max == 0 || xdev->user_max > MAX_USER_IRQ) + xdev->user_max = MAX_USER_IRQ; + if (xdev->h2c_channel_max == 0 || + xdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX) + xdev->h2c_channel_max = XDMA_CHANNEL_NUM_MAX; + if (xdev->c2h_channel_max == 0 || + xdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX) + xdev->c2h_channel_max = XDMA_CHANNEL_NUM_MAX; + + rv = pci_enable_device(pdev); + if (rv) { + dbg_init("pci_enable_device() failed, %d.\n", rv); + goto err_enable; + } + + /* keep INTx enabled */ + pci_check_intr_pend(pdev); + + /* enable relaxed ordering */ + pci_enable_relaxed_ordering(pdev); + + pci_check_extended_tag(xdev, pdev); + + /* force MRRS to be 512 */ + rv = pcie_set_readrq(pdev, 512); + if (rv) + pr_info("device %s, error set PCI_EXP_DEVCTL_READRQ: %d.\n", + dev_name(&pdev->dev), rv); + + /* enable bus master capability */ + pci_set_master(pdev); + + rv = request_regions(xdev, pdev); + if (rv) + goto err_regions; + + rv = map_bars(xdev, pdev); + if (rv) + goto err_map; + + rv = set_dma_mask(pdev); + if (rv) + goto err_mask; + + check_nonzero_interrupt_status(xdev); + /* explicitely zero all interrupt enable masks */ + channel_interrupts_disable(xdev, ~0); + user_interrupts_disable(xdev, ~0); + read_interrupts(xdev); + + rv = probe_engines(xdev); + if (rv) + goto err_engines; + + rv = enable_msi_msix(xdev, pdev); + if (rv < 0) + goto err_enable_msix; + + rv = irq_setup(xdev, pdev); + if (rv < 0) + goto err_interrupts; + + if (!poll_mode) + channel_interrupts_enable(xdev, ~0); + + /* Flush writes */ + read_interrupts(xdev); + + *user_max = xdev->user_max; + *h2c_channel_max = xdev->h2c_channel_max; + *c2h_channel_max = xdev->c2h_channel_max; + + xdma_device_flag_clear(xdev, XDEV_FLAG_OFFLINE); + return (void *)xdev; + +err_interrupts: + irq_teardown(xdev); +err_enable_msix: + disable_msi_msix(xdev, pdev); +err_engines: + remove_engines(xdev); +err_mask: + unmap_bars(xdev, pdev); +err_map: + if (xdev->got_regions) + pci_release_regions(pdev); +err_regions: + if (!xdev->regions_in_use) + pci_disable_device(pdev); +err_enable: + xdev_list_remove(xdev); + kfree(xdev); + return NULL; +} +EXPORT_SYMBOL_GPL(xdma_device_open); + +void xdma_device_close(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + dbg_init("pdev 0x%p, xdev 0x%p.\n", pdev, dev_hndl); + + if (!dev_hndl) + return; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return; + + dbg_sg("remove(dev = 0x%p) where pdev->dev.driver_data = 0x%p\n", + pdev, xdev); + if (xdev->pdev != pdev) { + dbg_sg("pci_dev(0x%lx) != pdev(0x%lx)\n", + (unsigned long)xdev->pdev, (unsigned long)pdev); + } + + channel_interrupts_disable(xdev, ~0); + user_interrupts_disable(xdev, ~0); + read_interrupts(xdev); + + irq_teardown(xdev); + disable_msi_msix(xdev, pdev); + + remove_engines(xdev); + unmap_bars(xdev, pdev); + + if (xdev->got_regions) { + dbg_init("pci_release_regions 0x%p.\n", pdev); + pci_release_regions(pdev); + } + + if (!xdev->regions_in_use) { + dbg_init("pci_disable_device 0x%p.\n", pdev); + pci_disable_device(pdev); + } + + xdev_list_remove(xdev); + + kfree(xdev); +} +EXPORT_SYMBOL_GPL(xdma_device_close); + +void xdma_device_offline(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + struct xdma_engine *engine; + int i; + + if (!dev_hndl) + return; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return; + +pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); + xdma_device_flag_set(xdev, XDEV_FLAG_OFFLINE); + + /* wait for all engines to be idle */ + for (i = 0; i < xdev->h2c_channel_max; i++) { + unsigned long flags; + + engine = &xdev->engine_h2c[i]; + + if (engine->magic == MAGIC_ENGINE) { + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown |= ENGINE_SHUTDOWN_REQUEST; + + xdma_engine_stop(engine); + engine->running = 0; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + for (i = 0; i < xdev->c2h_channel_max; i++) { + unsigned long flags; + + engine = &xdev->engine_c2h[i]; + if (engine->magic == MAGIC_ENGINE) { + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown |= ENGINE_SHUTDOWN_REQUEST; + + xdma_engine_stop(engine); + engine->running = 0; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + /* turn off interrupts */ + channel_interrupts_disable(xdev, ~0); + user_interrupts_disable(xdev, ~0); + read_interrupts(xdev); + irq_teardown(xdev); + + pr_info("xdev 0x%p, done.\n", xdev); +} +EXPORT_SYMBOL_GPL(xdma_device_offline); + +void xdma_device_online(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + struct xdma_engine *engine; + unsigned long flags; + int i; + + if (!dev_hndl) + return; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return; + +pr_info("pdev 0x%p, xdev 0x%p.\n", pdev, xdev); + + for (i = 0; i < xdev->h2c_channel_max; i++) { + engine = &xdev->engine_h2c[i]; + if (engine->magic == MAGIC_ENGINE) { + engine_init_regs(engine); + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown &= ~ENGINE_SHUTDOWN_REQUEST; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + for (i = 0; i < xdev->c2h_channel_max; i++) { + engine = &xdev->engine_c2h[i]; + if (engine->magic == MAGIC_ENGINE) { + engine_init_regs(engine); + spin_lock_irqsave(&engine->lock, flags); + engine->shutdown &= ~ENGINE_SHUTDOWN_REQUEST; + spin_unlock_irqrestore(&engine->lock, flags); + } + } + + /* re-write the interrupt table */ + if (!poll_mode) { + irq_setup(xdev, pdev); + + channel_interrupts_enable(xdev, ~0); + user_interrupts_enable(xdev, xdev->mask_irq_user); + read_interrupts(xdev); + } + + xdma_device_flag_clear(xdev, XDEV_FLAG_OFFLINE); +pr_info("xdev 0x%p, done.\n", xdev); +} +EXPORT_SYMBOL_GPL(xdma_device_online); + +int xdma_device_restart(struct pci_dev *pdev, void *dev_hndl) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, pdev, dev_hndl) < 0) + return -EINVAL; + + pr_info("NOT implemented, 0x%p.\n", xdev); + return -EINVAL; +} +EXPORT_SYMBOL_GPL(xdma_device_restart); + +int xdma_user_isr_register(void *dev_hndl, unsigned int mask, + irq_handler_t handler, void *dev) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + int i; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + for (i = 0; i < xdev->user_max && mask; i++) { + unsigned int bit = (1 << i); + + if ((bit & mask) == 0) + continue; + + mask &= ~bit; + xdev->user_irq[i].handler = handler; + xdev->user_irq[i].dev = dev; + } + + return 0; +} +EXPORT_SYMBOL_GPL(xdma_user_isr_register); + +int xdma_user_isr_enable(void *dev_hndl, unsigned int mask) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + xdev->mask_irq_user |= mask; + /* enable user interrupts */ + user_interrupts_enable(xdev, mask); + read_interrupts(xdev); + + return 0; +} +EXPORT_SYMBOL_GPL(xdma_user_isr_enable); + +int xdma_user_isr_disable(void *dev_hndl, unsigned int mask) +{ + struct xdma_dev *xdev = (struct xdma_dev *)dev_hndl; + + if (!dev_hndl) + return -EINVAL; + + if (debug_check_dev_hndl(__func__, xdev->pdev, dev_hndl) < 0) + return -EINVAL; + + xdev->mask_irq_user &= ~mask; + user_interrupts_disable(xdev, mask); + read_interrupts(xdev); + + return 0; +} +EXPORT_SYMBOL_GPL(xdma_user_isr_disable); + +#ifdef __LIBXDMA_MOD__ +static int __init xdma_base_init(void) +{ + printk(KERN_INFO "%s", version); + return 0; +} + +static void __exit xdma_base_exit(void) +{ + return; +} + +module_init(xdma_base_init); +module_exit(xdma_base_exit); +#endif +/* makes an existing transfer cyclic */ +static void xdma_transfer_cyclic(struct xdma_transfer *transfer) +{ + /* link last descriptor to first descriptor */ + xdma_desc_link(transfer->desc_virt + transfer->desc_num - 1, + transfer->desc_virt, transfer->desc_bus); + /* remember transfer is cyclic */ + transfer->cyclic = 1; +} + +static int transfer_monitor_cyclic(struct xdma_engine *engine, + struct xdma_transfer *transfer, int timeout_ms) +{ + struct xdma_result *result; + int rc = 0; + + BUG_ON(!engine); + BUG_ON(!transfer); + + result = engine->cyclic_result; + BUG_ON(!result); + + if (poll_mode) { + int i ; + for (i = 0; i < 5; i++) { + rc = engine_service_poll(engine, 0); + if (rc) { + pr_info("%s service_poll failed %d.\n", + engine->name, rc); + rc = -ERESTARTSYS; + } + if (result[engine->rx_head].status) + return 0; + } + } else { + if (enable_credit_mp){ + dbg_tfr("%s: rx_head=%d,rx_tail=%d, wait ...\n", + engine->name, engine->rx_head, engine->rx_tail); + rc = wait_event_interruptible_timeout( transfer->wq, + (engine->rx_head!=engine->rx_tail || + engine->rx_overrun), + msecs_to_jiffies(timeout_ms)); + dbg_tfr("%s: wait returns %d, rx %d/%d, overrun %d.\n", + engine->name, rc, engine->rx_head, + engine->rx_tail, engine->rx_overrun); + } else { + rc = wait_event_interruptible_timeout( transfer->wq, + engine->eop_found, + msecs_to_jiffies(timeout_ms)); + dbg_tfr("%s: wait returns %d, eop_found %d.\n", + engine->name, rc, engine->eop_found); + } + } + + return 0; +} + +struct scatterlist *sglist_index(struct sg_table *sgt, unsigned int idx) +{ + struct scatterlist *sg = sgt->sgl; + int i; + + if (idx >= sgt->orig_nents) + return NULL; + + if (!idx) + return sg; + + for (i = 0; i < idx; i++, sg = sg_next(sg)) + ; + + return sg; +} + +static int copy_cyclic_to_user(struct xdma_engine *engine, int pkt_length, + int head, char __user *buf, size_t count) +{ + struct scatterlist *sg; + int more = pkt_length; + + BUG_ON(!engine); + BUG_ON(!buf); + + dbg_tfr("%s, pkt_len %d, head %d, user buf idx %u.\n", + engine->name, pkt_length, head, engine->user_buffer_index); + + sg = sglist_index(&engine->cyclic_sgt, head); + if (!sg) { + pr_info("%s, head %d OOR, sgl %u.\n", + engine->name, head, engine->cyclic_sgt.orig_nents); + return -EIO; + } + + /* EOP found? Transfer anything from head to EOP */ + while (more) { + unsigned int copy = more > PAGE_SIZE ? PAGE_SIZE : more; + unsigned int blen = count - engine->user_buffer_index; + int rv; + + if (copy > blen) + copy = blen; + + dbg_tfr("%s sg %d, 0x%p, copy %u to user %u.\n", + engine->name, head, sg, copy, + engine->user_buffer_index); + + rv = copy_to_user(&buf[engine->user_buffer_index], + page_address(sg_page(sg)), copy); + if (rv) { + pr_info("%s copy_to_user %u failed %d\n", + engine->name, copy, rv); + return -EIO; + } + + more -= copy; + engine->user_buffer_index += copy; + + if (engine->user_buffer_index == count) { + /* user buffer used up */ + break; + } + + head++; + if (head >= CYCLIC_RX_PAGES_MAX) { + head = 0; + sg = engine->cyclic_sgt.sgl; + } else + sg = sg_next(sg); + } + + return pkt_length; +} + +static int complete_cyclic(struct xdma_engine *engine, char __user *buf, + size_t count) +{ + struct xdma_result *result; + int pkt_length = 0; + int fault = 0; + int eop = 0; + int head; + int rc = 0; + int num_credit = 0; + unsigned long flags; + + BUG_ON(!engine); + result = engine->cyclic_result; + BUG_ON(!result); + + spin_lock_irqsave(&engine->lock, flags); + + /* where the host currently is in the ring buffer */ + head = engine->rx_head; + + /* iterate over newly received results */ + while (engine->rx_head != engine->rx_tail||engine->rx_overrun) { + + WARN_ON(result[engine->rx_head].status==0); + + dbg_tfr("%s, result[%d].status = 0x%x length = 0x%x.\n", + engine->name, engine->rx_head, + result[engine->rx_head].status, + result[engine->rx_head].length); + + if ((result[engine->rx_head].status >> 16) != C2H_WB) { + pr_info("%s, result[%d].status 0x%x, no magic.\n", + engine->name, engine->rx_head, + result[engine->rx_head].status); + fault = 1; + } else if (result[engine->rx_head].length > PAGE_SIZE) { + pr_info("%s, result[%d].len 0x%x, > PAGE_SIZE 0x%lx.\n", + engine->name, engine->rx_head, + result[engine->rx_head].length, PAGE_SIZE); + fault = 1; + } else if (result[engine->rx_head].length == 0) { + pr_info("%s, result[%d].length 0x%x.\n", + engine->name, engine->rx_head, + result[engine->rx_head].length); + fault = 1; + /* valid result */ + } else { + pkt_length += result[engine->rx_head].length; + num_credit++; + /* seen eop? */ + //if (result[engine->rx_head].status & RX_STATUS_EOP) + if (result[engine->rx_head].status & RX_STATUS_EOP){ + eop = 1; + engine->eop_found = 1; + } + + dbg_tfr("%s, pkt_length=%d (%s)\n", + engine->name, pkt_length, + eop ? "with EOP" : "no EOP yet"); + } + /* clear result */ + result[engine->rx_head].status = 0; + result[engine->rx_head].length = 0; + /* proceed head pointer so we make progress, even when fault */ + engine->rx_head = (engine->rx_head + 1) % CYCLIC_RX_PAGES_MAX; + + /* stop processing if a fault/eop was detected */ + if (fault || eop){ + break; + } + } + + spin_unlock_irqrestore(&engine->lock, flags); + + if (fault) + return -EIO; + + rc = copy_cyclic_to_user(engine, pkt_length, head, buf, count); + engine->rx_overrun = 0; + /* if copy is successful, release credits */ + if(rc > 0) + write_register(num_credit,&engine->sgdma_regs->credits, 0); + + return rc; +} + +ssize_t xdma_engine_read_cyclic(struct xdma_engine *engine, char __user *buf, + size_t count, int timeout_ms) +{ + int i = 0; + int rc = 0; + int rc_len = 0; + struct xdma_transfer *transfer; + + BUG_ON(!engine); + BUG_ON(engine->magic != MAGIC_ENGINE); + + transfer = &engine->cyclic_req->xfer; + BUG_ON(!transfer); + + engine->user_buffer_index = 0; + + do { + rc = transfer_monitor_cyclic(engine, transfer, timeout_ms); + if (rc < 0) + return rc; + rc = complete_cyclic(engine, buf, count); + if (rc < 0) + return rc; + rc_len += rc; + + i++; + if (i > 10) + break; + } while (!engine->eop_found); + + if(enable_credit_mp) + engine->eop_found = 0; + + return rc_len; +} + +static void sgt_free_with_pages(struct sg_table *sgt, int dir, + struct pci_dev *pdev) +{ + struct scatterlist *sg = sgt->sgl; + int npages = sgt->orig_nents; + int i; + + for (i = 0; i < npages; i++, sg = sg_next(sg)) { + struct page *pg = sg_page(sg); + dma_addr_t bus = sg_dma_address(sg); + + if (pg) { + if (pdev) + pci_unmap_page(pdev, bus, PAGE_SIZE, dir); + __free_page(pg); + } else + break; + } + sg_free_table(sgt); + memset(sgt, 0, sizeof(struct sg_table)); +} + +static int sgt_alloc_with_pages(struct sg_table *sgt, unsigned int npages, + int dir, struct pci_dev *pdev) +{ + struct scatterlist *sg; + int i; + + if (sg_alloc_table(sgt, npages, GFP_KERNEL)) { + pr_info("sgt OOM.\n"); + return -ENOMEM; + } + + sg = sgt->sgl; + for (i = 0; i < npages; i++, sg = sg_next(sg)) { + struct page *pg = alloc_page(GFP_KERNEL); + + if (!pg) { + pr_info("%d/%u, page OOM.\n", i, npages); + goto err_out; + } + + if (pdev) { + dma_addr_t bus = pci_map_page(pdev, pg, 0, PAGE_SIZE, + dir); + if (unlikely(pci_dma_mapping_error(pdev, bus))) { + pr_info("%d/%u, page 0x%p map err.\n", + i, npages, pg); + __free_page(pg); + goto err_out; + } + sg_dma_address(sg) = bus; + sg_dma_len(sg) = PAGE_SIZE; + } + sg_set_page(sg, pg, PAGE_SIZE, 0); + } + + sgt->orig_nents = sgt->nents = npages; + + return 0; + +err_out: + sgt_free_with_pages(sgt, dir, pdev); + return -ENOMEM; +} + +/* + * !NOTE! reference/demo purpose only + * xdma_cyclic_transfer_setup is used for streaming C2H transfers: + * - A list of buffers are pre-allocated for incoming streaming data + * - the ring of the buffers is allowed to wrap around + */ +int xdma_cyclic_transfer_setup(struct xdma_engine *engine) +{ + struct xdma_dev *xdev; + struct xdma_transfer *xfer; + dma_addr_t bus; + unsigned long flags; + int i; + int rc; + + BUG_ON(!engine); + xdev = engine->xdev; + BUG_ON(!xdev); + + if (engine->cyclic_req) { + //pr_info("%s: exclusive access already taken.\n", + //engine->name); + return -EBUSY; + } + + spin_lock_irqsave(&engine->lock, flags); + + engine->rx_tail = 0; + engine->rx_head = 0; + engine->rx_overrun = 0; + engine->eop_found = 0; + + rc = sgt_alloc_with_pages(&engine->cyclic_sgt, CYCLIC_RX_PAGES_MAX, + engine->dir, xdev->pdev); + if (rc < 0) { + pr_info("%s cyclic pages %u OOM.\n", + engine->name, CYCLIC_RX_PAGES_MAX); + goto err_out; + } + + engine->cyclic_req = xdma_init_request(&engine->cyclic_sgt, 0); + if (!engine->cyclic_req) { + pr_info("%s cyclic request OOM.\n", engine->name); + rc = -ENOMEM; + goto err_out; + } + +#ifdef __LIBXDMA_DEBUG__ + xdma_request_cb_dump(engine->cyclic_req); +#endif + + rc = transfer_init(engine, engine->cyclic_req); + if (rc < 0) + goto err_out; + + xfer = &engine->cyclic_req->xfer; + + /* replace source addresses with result write-back addresses */ + memset(engine->cyclic_result, 0, + CYCLIC_RX_PAGES_MAX * sizeof(struct xdma_result)); + bus = engine->cyclic_result_bus; + for (i = 0; i < xfer->desc_num; i++) { + xfer->desc_virt[i].src_addr_lo = cpu_to_le32(PCI_DMA_L(bus)); + xfer->desc_virt[i].src_addr_hi = cpu_to_le32(PCI_DMA_H(bus)); + bus += sizeof(struct xdma_result); + } + /* set control of all descriptors */ + for (i = 0; i < xfer->desc_num; i++) { + xdma_desc_control_clear(xfer->desc_virt + i, LS_BYTE_MASK); + xdma_desc_control_set(xfer->desc_virt + i, + XDMA_DESC_EOP | XDMA_DESC_COMPLETED); + } + + /* make this a cyclic transfer */ + xdma_transfer_cyclic(xfer); + +#ifdef __LIBXDMA_DEBUG__ + transfer_dump(xfer); +#endif + + if(enable_credit_mp){ + //write_register(RX_BUF_PAGES,&engine->sgdma_regs->credits); + write_register(128, &engine->sgdma_regs->credits, 0); + } + + spin_unlock_irqrestore(&engine->lock, flags); + + /* start cyclic transfer */ + transfer_queue(engine, xfer); + + return 0; + + /* unwind on errors */ +err_out: + if (engine->cyclic_req) { + xdma_request_free(engine->cyclic_req); + engine->cyclic_req = NULL; + } + + if (engine->cyclic_sgt.orig_nents) { + sgt_free_with_pages(&engine->cyclic_sgt, engine->dir, + xdev->pdev); + engine->cyclic_sgt.orig_nents = 0; + engine->cyclic_sgt.nents = 0; + engine->cyclic_sgt.sgl = NULL; + } + + spin_unlock_irqrestore(&engine->lock, flags); + + return rc; +} + + +static int cyclic_shutdown_polled(struct xdma_engine *engine) +{ + BUG_ON(!engine); + + spin_lock(&engine->lock); + + dbg_tfr("Polling for shutdown completion\n"); + do { + engine_status_read(engine, 1, 0); + schedule(); + } while (engine->status & XDMA_STAT_BUSY); + + if ((engine->running) && !(engine->status & XDMA_STAT_BUSY)) { + dbg_tfr("Engine has stopped\n"); + + if (!list_empty(&engine->transfer_list)) + engine_transfer_dequeue(engine); + + engine_service_shutdown(engine); + } + + dbg_tfr("Shutdown completion polling done\n"); + spin_unlock(&engine->lock); + + return 0; +} + +static int cyclic_shutdown_interrupt(struct xdma_engine *engine) +{ + int rc; + + BUG_ON(!engine); + + rc = wait_event_interruptible_timeout(engine->shutdown_wq, + !engine->running, msecs_to_jiffies(10000)); + +#if 0 + if (rc) { + dbg_tfr("wait_event_interruptible=%d\n", rc); + return rc; + } +#endif + + if (engine->running) { + pr_info("%s still running?!, %d\n", engine->name, rc); + return -EINVAL; + } + + return rc; +} + +int xdma_cyclic_transfer_teardown(struct xdma_engine *engine) +{ + int rc; + struct xdma_dev *xdev = engine->xdev; + struct xdma_transfer *transfer; + unsigned long flags; + + transfer = engine_cyclic_stop(engine); + + spin_lock_irqsave(&engine->lock, flags); + if (transfer) { + dbg_tfr("%s: stop transfer 0x%p.\n", engine->name, transfer); + if (transfer != &engine->cyclic_req->xfer) { + pr_info("%s unexpected transfer 0x%p/0x%p\n", + engine->name, transfer, + &engine->cyclic_req->xfer); + } + } + /* allow engine to be serviced after stop request */ + spin_unlock_irqrestore(&engine->lock, flags); + + /* wait for engine to be no longer running */ + if (poll_mode) + rc = cyclic_shutdown_polled(engine); + else + rc = cyclic_shutdown_interrupt(engine); + + /* obtain spin lock to atomically remove resources */ + spin_lock_irqsave(&engine->lock, flags); + + if (engine->cyclic_req) { + xdma_request_free(engine->cyclic_req); + engine->cyclic_req = NULL; + } + + if (engine->cyclic_sgt.orig_nents) { + sgt_free_with_pages(&engine->cyclic_sgt, engine->dir, + xdev->pdev); + engine->cyclic_sgt.orig_nents = 0; + engine->cyclic_sgt.nents = 0; + engine->cyclic_sgt.sgl = NULL; + } + + spin_unlock_irqrestore(&engine->lock, flags); + + return 0; +} + +int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg) +{ + int rv; + unsigned long dst; + u32 w = XDMA_CTRL_NON_INCR_ADDR; + + dbg_perf("IOCTL_XDMA_ADDRMODE_SET\n"); + rv = get_user(dst, (int __user *)arg); + + if (rv == 0) { + engine->non_incr_addr = !!dst; + if (engine->non_incr_addr) + write_register(w, &engine->regs->control_w1s, + (unsigned long)(&engine->regs->control_w1s) - + (unsigned long)(&engine->regs)); + else + write_register(w, &engine->regs->control_w1c, + (unsigned long)(&engine->regs->control_w1c) - + (unsigned long)(&engine->regs)); + } + engine_alignments(engine); + + return rv; +} + diff --git a/sources/xdma_driver/xdma/libxdma.h b/sources/xdma_driver/xdma/libxdma.h new file mode 100644 index 0000000..bbf1a45 --- /dev/null +++ b/sources/xdma_driver/xdma/libxdma.h @@ -0,0 +1,601 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef XDMA_LIB_H +#define XDMA_LIB_H + +#include <linux/version.h> +#include <linux/types.h> +#include <linux/uaccess.h> +#include <linux/module.h> +#include <linux/dma-mapping.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/jiffies.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/workqueue.h> + +/* Switch debug printing on/off */ +#define XDMA_DEBUG 0 + +/* SECTION: Preprocessor macros/constants */ +#define XDMA_BAR_NUM (6) + +/* maximum amount of register space to map */ +#define XDMA_BAR_SIZE (0x8000UL) + +/* Use this definition to poll several times between calls to schedule */ +#define NUM_POLLS_PER_SCHED 100 + +#define XDMA_CHANNEL_NUM_MAX (4) +/* + * interrupts per engine, rad2_vul.sv:237 + * .REG_IRQ_OUT (reg_irq_from_ch[(channel*2) +: 2]), + */ +#define XDMA_ENG_IRQ_NUM (1) +#define MAX_EXTRA_ADJ (15) +#define RX_STATUS_EOP (1) + +/* Target internal components on XDMA control BAR */ +#define XDMA_OFS_INT_CTRL (0x2000UL) +#define XDMA_OFS_CONFIG (0x3000UL) + +/* maximum number of desc per transfer request */ +#define XDMA_TRANSFER_MAX_DESC (2048) + +/* maximum size of a single DMA transfer descriptor */ +#define XDMA_DESC_BLEN_BITS 28 +#define XDMA_DESC_BLEN_MAX ((1 << (XDMA_DESC_BLEN_BITS)) - 1) + +/* bits of the SG DMA control register */ +#define XDMA_CTRL_RUN_STOP (1UL << 0) +#define XDMA_CTRL_IE_DESC_STOPPED (1UL << 1) +#define XDMA_CTRL_IE_DESC_COMPLETED (1UL << 2) +#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH (1UL << 3) +#define XDMA_CTRL_IE_MAGIC_STOPPED (1UL << 4) +#define XDMA_CTRL_IE_IDLE_STOPPED (1UL << 6) +#define XDMA_CTRL_IE_READ_ERROR (0x1FUL << 9) +#define XDMA_CTRL_IE_DESC_ERROR (0x1FUL << 19) +#define XDMA_CTRL_NON_INCR_ADDR (1UL << 25) +#define XDMA_CTRL_POLL_MODE_WB (1UL << 26) + +/* bits of the SG DMA status register */ +#define XDMA_STAT_BUSY (1UL << 0) +#define XDMA_STAT_DESC_STOPPED (1UL << 1) +#define XDMA_STAT_DESC_COMPLETED (1UL << 2) +#define XDMA_STAT_ALIGN_MISMATCH (1UL << 3) +#define XDMA_STAT_MAGIC_STOPPED (1UL << 4) +#define XDMA_STAT_INVALID_LEN (1UL << 5) +#define XDMA_STAT_IDLE_STOPPED (1UL << 6) + +#define XDMA_STAT_COMMON_ERR_MASK \ + (XDMA_STAT_ALIGN_MISMATCH | XDMA_STAT_MAGIC_STOPPED | \ + XDMA_STAT_INVALID_LEN) + +/* desc_error, C2H & H2C */ +#define XDMA_STAT_DESC_UNSUPP_REQ (1UL << 19) +#define XDMA_STAT_DESC_COMPL_ABORT (1UL << 20) +#define XDMA_STAT_DESC_PARITY_ERR (1UL << 21) +#define XDMA_STAT_DESC_HEADER_EP (1UL << 22) +#define XDMA_STAT_DESC_UNEXP_COMPL (1UL << 23) + +#define XDMA_STAT_DESC_ERR_MASK \ + (XDMA_STAT_DESC_UNSUPP_REQ | XDMA_STAT_DESC_COMPL_ABORT | \ + XDMA_STAT_DESC_PARITY_ERR | XDMA_STAT_DESC_HEADER_EP | \ + XDMA_STAT_DESC_UNEXP_COMPL) + +/* read error: H2C */ +#define XDMA_STAT_H2C_R_UNSUPP_REQ (1UL << 9) +#define XDMA_STAT_H2C_R_COMPL_ABORT (1UL << 10) +#define XDMA_STAT_H2C_R_PARITY_ERR (1UL << 11) +#define XDMA_STAT_H2C_R_HEADER_EP (1UL << 12) +#define XDMA_STAT_H2C_R_UNEXP_COMPL (1UL << 13) + +#define XDMA_STAT_H2C_R_ERR_MASK \ + (XDMA_STAT_H2C_R_UNSUPP_REQ | XDMA_STAT_H2C_R_COMPL_ABORT | \ + XDMA_STAT_H2C_R_PARITY_ERR | XDMA_STAT_H2C_R_HEADER_EP | \ + XDMA_STAT_H2C_R_UNEXP_COMPL) + +/* write error, H2C only */ +#define XDMA_STAT_H2C_W_DECODE_ERR (1UL << 14) +#define XDMA_STAT_H2C_W_SLAVE_ERR (1UL << 15) + +#define XDMA_STAT_H2C_W_ERR_MASK \ + (XDMA_STAT_H2C_W_DECODE_ERR | XDMA_STAT_H2C_W_SLAVE_ERR) + +/* read error: C2H */ +#define XDMA_STAT_C2H_R_DECODE_ERR (1UL << 9) +#define XDMA_STAT_C2H_R_SLAVE_ERR (1UL << 10) + +#define XDMA_STAT_C2H_R_ERR_MASK \ + (XDMA_STAT_C2H_R_DECODE_ERR | XDMA_STAT_C2H_R_SLAVE_ERR) + +/* all combined */ +#define XDMA_STAT_H2C_ERR_MASK \ + (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \ + XDMA_STAT_H2C_R_ERR_MASK | XDMA_STAT_H2C_W_ERR_MASK) + +#define XDMA_STAT_C2H_ERR_MASK \ + (XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \ + XDMA_STAT_C2H_R_ERR_MASK) + +/* bits of the SGDMA descriptor control field */ +#define XDMA_DESC_STOPPED (1UL << 0) +#define XDMA_DESC_COMPLETED (1UL << 1) +#define XDMA_DESC_EOP (1UL << 4) + +#define XDMA_PERF_RUN (1UL << 0) +#define XDMA_PERF_CLEAR (1UL << 1) +#define XDMA_PERF_AUTO (1UL << 2) + +#define MAGIC_ENGINE 0xEEEEEEEEUL +#define MAGIC_DEVICE 0xDDDDDDDDUL + +/* upper 16-bits of engine identifier register */ +#define XDMA_ID_H2C 0x1fc0U +#define XDMA_ID_C2H 0x1fc1U + +/* for C2H AXI-ST mode */ +#define CYCLIC_RX_PAGES_MAX 256 + +#define LS_BYTE_MASK 0x000000FFUL + +#define BLOCK_ID_MASK 0xFFF00000 +#define BLOCK_ID_HEAD 0x1FC00000 + +#define IRQ_BLOCK_ID 0x1fc20000UL +#define CONFIG_BLOCK_ID 0x1fc30000UL + +#define WB_COUNT_MASK 0x00ffffffUL +#define WB_ERR_MASK (1UL << 31) +#define POLL_TIMEOUT_SECONDS 10 + +#define MAX_USER_IRQ 16 + +#define MAX_DESC_BUS_ADDR (0xffffffffULL) + +#define DESC_MAGIC 0xAD4B0000UL + +#define C2H_WB 0x52B4UL + +#define MAX_NUM_ENGINES (XDMA_CHANNEL_NUM_MAX * 2) +#define H2C_CHANNEL_OFFSET 0x1000 +#define SGDMA_OFFSET_FROM_CHANNEL 0x4000 +#define CHANNEL_SPACING 0x100 +#define TARGET_SPACING 0x1000 + +#define BYPASS_MODE_SPACING 0x0100 + +/* obtain the 32 most significant (high) bits of a 32-bit or 64-bit address */ +#define PCI_DMA_H(addr) ((addr >> 16) >> 16) +/* obtain the 32 least significant (low) bits of a 32-bit or 64-bit address */ +#define PCI_DMA_L(addr) (addr & 0xffffffffUL) + +#ifndef VM_RESERVED + #define VMEM_FLAGS (VM_IO | VM_DONTEXPAND | VM_DONTDUMP) +#else + #define VMEM_FLAGS (VM_IO | VM_RESERVED) +#endif + +#ifdef __LIBXDMA_DEBUG__ +#define dbg_io pr_err +#define dbg_fops pr_err +#define dbg_perf pr_err +#define dbg_sg pr_err +#define dbg_tfr pr_err +#define dbg_irq pr_err +#define dbg_init pr_err +#define dbg_desc pr_err +#else +/* disable debugging */ +#define dbg_io(...) +#define dbg_fops(...) +#define dbg_perf(...) +#define dbg_sg(...) +#define dbg_tfr(...) +#define dbg_irq(...) +#define dbg_init(...) +#define dbg_desc(...) +#endif + +/* SECTION: Enum definitions */ +enum transfer_state { + TRANSFER_STATE_NEW = 0, + TRANSFER_STATE_SUBMITTED, + TRANSFER_STATE_COMPLETED, + TRANSFER_STATE_FAILED, + TRANSFER_STATE_ABORTED +}; + +enum shutdown_state { + ENGINE_SHUTDOWN_NONE = 0, /* No shutdown in progress */ + ENGINE_SHUTDOWN_REQUEST = 1, /* engine requested to shutdown */ + ENGINE_SHUTDOWN_IDLE = 2 /* engine has shutdown and is idle */ +}; + +enum dev_capabilities { + CAP_64BIT_DMA = 2, + CAP_64BIT_DESC = 4, + CAP_ENGINE_WRITE = 8, + CAP_ENGINE_READ = 16 +}; + +/* SECTION: Structure definitions */ + +struct config_regs { + u32 identifier; + u32 reserved_1[4]; + u32 msi_enable; +}; + +/** + * SG DMA Controller status and control registers + * + * These registers make the control interface for DMA transfers. + * + * It sits in End Point (FPGA) memory BAR[0] for 32-bit or BAR[0:1] for 64-bit. + * It references the first descriptor which exists in Root Complex (PC) memory. + * + * @note The registers must be accessed using 32-bit (PCI DWORD) read/writes, + * and their values are in little-endian byte ordering. + */ +struct engine_regs { + u32 identifier; + u32 control; + u32 control_w1s; + u32 control_w1c; + u32 reserved_1[12]; /* padding */ + + u32 status; + u32 status_rc; + u32 completed_desc_count; + u32 alignments; + u32 reserved_2[14]; /* padding */ + + u32 poll_mode_wb_lo; + u32 poll_mode_wb_hi; + u32 interrupt_enable_mask; + u32 interrupt_enable_mask_w1s; + u32 interrupt_enable_mask_w1c; + u32 reserved_3[9]; /* padding */ + + u32 perf_ctrl; + u32 perf_cyc_lo; + u32 perf_cyc_hi; + u32 perf_dat_lo; + u32 perf_dat_hi; + u32 perf_pnd_lo; + u32 perf_pnd_hi; +} __packed; + +struct engine_sgdma_regs { + u32 identifier; + u32 reserved_1[31]; /* padding */ + + /* bus address to first descriptor in Root Complex Memory */ + u32 first_desc_lo; + u32 first_desc_hi; + /* number of adjacent descriptors at first_desc */ + u32 first_desc_adjacent; + u32 credits; +} __packed; + +struct msix_vec_table_entry { + u32 msi_vec_addr_lo; + u32 msi_vec_addr_hi; + u32 msi_vec_data_lo; + u32 msi_vec_data_hi; +} __packed; + +struct msix_vec_table { + struct msix_vec_table_entry entry_list[32]; +} __packed; + +struct interrupt_regs { + u32 identifier; + u32 user_int_enable; + u32 user_int_enable_w1s; + u32 user_int_enable_w1c; + u32 channel_int_enable; + u32 channel_int_enable_w1s; + u32 channel_int_enable_w1c; + u32 reserved_1[9]; /* padding */ + + u32 user_int_request; + u32 channel_int_request; + u32 user_int_pending; + u32 channel_int_pending; + u32 reserved_2[12]; /* padding */ + + u32 user_msi_vector[8]; + u32 channel_msi_vector[8]; +} __packed; + +struct sgdma_common_regs { + u32 padding[8]; + u32 credit_mode_enable; + u32 credit_mode_enable_w1s; + u32 credit_mode_enable_w1c; +} __packed; + + +/* Structure for polled mode descriptor writeback */ +struct xdma_poll_wb { + u32 completed_desc_count; + u32 reserved_1[7]; +} __packed; + + +/** + * Descriptor for a single contiguous memory block transfer. + * + * Multiple descriptors are linked by means of the next pointer. An additional + * extra adjacent number gives the amount of extra contiguous descriptors. + * + * The descriptors are in root complex memory, and the bytes in the 32-bit + * words must be in little-endian byte ordering. + */ +struct xdma_desc { + u32 control; + u32 bytes; /* transfer length in bytes */ + u32 src_addr_lo; /* source address (low 32-bit) */ + u32 src_addr_hi; /* source address (high 32-bit) */ + u32 dst_addr_lo; /* destination address (low 32-bit) */ + u32 dst_addr_hi; /* destination address (high 32-bit) */ + /* + * next descriptor in the single-linked list of descriptors; + * this is the PCIe (bus) address of the next descriptor in the + * root complex memory + */ + u32 next_lo; /* next desc address (low 32-bit) */ + u32 next_hi; /* next desc address (high 32-bit) */ +} __packed; + +/* 32 bytes (four 32-bit words) or 64 bytes (eight 32-bit words) */ +struct xdma_result { + u32 status; + u32 length; + u32 reserved_1[6]; /* padding */ +} __packed; + +struct sw_desc { + dma_addr_t addr; + unsigned int len; +}; + +/* Describes a (SG DMA) single transfer for the engine */ +struct xdma_transfer { + struct list_head entry; /* queue of non-completed transfers */ + struct xdma_desc *desc_virt; /* virt addr of the 1st descriptor */ + dma_addr_t desc_bus; /* bus addr of the first descriptor */ + int desc_adjacent; /* adjacent descriptors at desc_bus */ + int desc_num; /* number of descriptors in transfer */ + enum dma_data_direction dir; + wait_queue_head_t wq; /* wait queue for transfer completion */ + + enum transfer_state state; /* state of the transfer */ + unsigned int flags; +#define XFER_FLAG_NEED_UNMAP 0x1 + int cyclic; /* flag if transfer is cyclic */ + int last_in_request; /* flag if last within request */ + unsigned int len; + struct sg_table *sgt; +}; + +struct xdma_request_cb { + struct sg_table *sgt; + unsigned int total_len; + u64 ep_addr; + + struct xdma_transfer xfer; + + unsigned int sw_desc_idx; + unsigned int sw_desc_cnt; + struct sw_desc sdesc[0]; +}; + +struct xdma_engine { + unsigned long magic; /* structure ID for sanity checks */ + struct xdma_dev *xdev; /* parent device */ + char name[5]; /* name of this engine */ + int version; /* version of this engine */ + //dev_t cdevno; /* character device major:minor */ + //struct cdev cdev; /* character device (embedded struct) */ + + /* HW register address offsets */ + struct engine_regs *regs; /* Control reg BAR offset */ + struct engine_sgdma_regs *sgdma_regs; /* SGDAM reg BAR offset */ + u32 bypass_offset; /* Bypass mode BAR offset */ + + /* Engine state, configuration and flags */ + enum shutdown_state shutdown; /* engine shutdown mode */ + enum dma_data_direction dir; + int device_open; /* flag if engine node open, ST mode only */ + int running; /* flag if the driver started engine */ + int non_incr_addr; /* flag if non-incremental addressing used */ + int streaming; + int addr_align; /* source/dest alignment in bytes */ + int len_granularity; /* transfer length multiple */ + int addr_bits; /* HW datapath address width */ + int channel; /* engine indices */ + int max_extra_adj; /* descriptor prefetch capability */ + int desc_dequeued; /* num descriptors of completed transfers */ + u32 status; /* last known status of device */ + u32 interrupt_enable_mask_value;/* only used for MSIX mode to store per-engine interrupt mask value */ + + /* Transfer list management */ + struct list_head transfer_list; /* queue of transfers */ + + /* Members applicable to AXI-ST C2H (cyclic) transfers */ + struct xdma_result *cyclic_result; + dma_addr_t cyclic_result_bus; /* bus addr for transfer */ + struct xdma_request_cb *cyclic_req; + struct sg_table cyclic_sgt; + u8 eop_found; /* used only for cyclic(rx:c2h) */ + + int rx_tail; /* follows the HW */ + int rx_head; /* where the SW reads from */ + int rx_overrun; /* flag if overrun occured */ + + /* for copy from cyclic buffer to user buffer */ + unsigned int user_buffer_index; + + /* Members associated with polled mode support */ + u8 *poll_mode_addr_virt; /* virt addr for descriptor writeback */ + dma_addr_t poll_mode_bus; /* bus addr for descriptor writeback */ + + /* Members associated with interrupt mode support */ + wait_queue_head_t shutdown_wq; /* wait queue for shutdown sync */ + spinlock_t lock; /* protects concurrent access */ + int prev_cpu; /* remember CPU# of (last) locker */ + int msix_irq_line; /* MSI-X vector for this engine */ + u32 irq_bitmask; /* IRQ bit mask for this engine */ + struct work_struct work; /* Work queue for interrupt handling */ + + spinlock_t desc_lock; /* protects concurrent access */ + dma_addr_t desc_bus; + struct xdma_desc *desc; + + /* for performance test support */ + struct xdma_performance_ioctl *xdma_perf; /* perf test control */ + wait_queue_head_t xdma_perf_wq; /* Perf test sync */ +}; + +struct xdma_user_irq { + struct xdma_dev *xdev; /* parent device */ + u8 user_idx; /* 0 ~ 15 */ + u8 events_irq; /* accumulated IRQs */ + spinlock_t events_lock; /* lock to safely update events_irq */ + wait_queue_head_t events_wq; /* wait queue to sync waiting threads */ + irq_handler_t handler; + + void *dev; +}; + +/* XDMA PCIe device specific book-keeping */ +#define XDEV_FLAG_OFFLINE 0x1 +struct xdma_dev { + struct list_head list_head; + struct list_head rcu_node; + + unsigned long magic; /* structure ID for sanity checks */ + struct pci_dev *pdev; /* pci device struct from probe() */ + int idx; /* dev index */ + + const char *mod_name; /* name of module owning the dev */ + + spinlock_t lock; /* protects concurrent access */ + unsigned int flags; + + /* PCIe BAR management */ + void *__iomem bar[XDMA_BAR_NUM]; /* addresses for mapped BARs */ + int user_bar_idx; /* BAR index of user logic */ + int config_bar_idx; /* BAR index of XDMA config logic */ + int bypass_bar_idx; /* BAR index of XDMA bypass logic */ + int regions_in_use; /* flag if dev was in use during probe() */ + int got_regions; /* flag if probe() obtained the regions */ + + int user_max; + int c2h_channel_max; + int h2c_channel_max; + + /* Interrupt management */ + int irq_count; /* interrupt counter */ + int irq_line; /* flag if irq allocated successfully */ + int msi_enabled; /* flag if msi was enabled for the device */ + int msix_enabled; /* flag if msi-x was enabled for the device */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4,12,0) + struct msix_entry entry[32]; /* msi-x vector/entry table */ +#endif + struct xdma_user_irq user_irq[16]; /* user IRQ management */ + unsigned int mask_irq_user; + + /* XDMA engine management */ + int engines_num; /* Total engine count */ + u32 mask_irq_h2c; + u32 mask_irq_c2h; + struct xdma_engine engine_h2c[XDMA_CHANNEL_NUM_MAX]; + struct xdma_engine engine_c2h[XDMA_CHANNEL_NUM_MAX]; + + /* SD_Accel specific */ + enum dev_capabilities capabilities; + u64 feature_id; +}; + +static inline int xdma_device_flag_check(struct xdma_dev *xdev, unsigned int f) +{ + unsigned long flags; + + spin_lock_irqsave(&xdev->lock, flags); + if (xdev->flags & f) { + spin_unlock_irqrestore(&xdev->lock, flags); + return 1; + } + spin_unlock_irqrestore(&xdev->lock, flags); + return 0; +} + +static inline int xdma_device_flag_test_n_set(struct xdma_dev *xdev, + unsigned int f) +{ + unsigned long flags; + int rv = 0; + + spin_lock_irqsave(&xdev->lock, flags); + if (xdev->flags & f) { + spin_unlock_irqrestore(&xdev->lock, flags); + rv = 1; + } else + xdev->flags |= f; + spin_unlock_irqrestore(&xdev->lock, flags); + return rv; +} + +static inline void xdma_device_flag_set(struct xdma_dev *xdev, unsigned int f) +{ + unsigned long flags; + + spin_lock_irqsave(&xdev->lock, flags); + xdev->flags |= f; + spin_unlock_irqrestore(&xdev->lock, flags); +} + +static inline void xdma_device_flag_clear(struct xdma_dev *xdev, unsigned int f) +{ + unsigned long flags; + + spin_lock_irqsave(&xdev->lock, flags); + xdev->flags &= ~f; + spin_unlock_irqrestore(&xdev->lock, flags); +} + +void write_register(u32 value, void *iomem); +u32 read_register(void *iomem); + +struct xdma_dev *xdev_find_by_pdev(struct pci_dev *pdev); + +void xdma_device_offline(struct pci_dev *pdev, void *dev_handle); +void xdma_device_online(struct pci_dev *pdev, void *dev_handle); + +int xdma_performance_submit(struct xdma_dev *xdev, struct xdma_engine *engine); +struct xdma_transfer *engine_cyclic_stop(struct xdma_engine *engine); +void enable_perf(struct xdma_engine *engine); +void get_perf_stats(struct xdma_engine *engine); + +int xdma_cyclic_transfer_setup(struct xdma_engine *engine); +int xdma_cyclic_transfer_teardown(struct xdma_engine *engine); +ssize_t xdma_engine_read_cyclic(struct xdma_engine *, char __user *, size_t, + int); +int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg); + +#endif /* XDMA_LIB_H */ diff --git a/sources/xdma_driver/xdma/libxdma.o.ur-safe b/sources/xdma_driver/xdma/libxdma.o.ur-safe new file mode 100644 index 0000000..699074b --- /dev/null +++ b/sources/xdma_driver/xdma/libxdma.o.ur-safe @@ -0,0 +1,10 @@ +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-29fb +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-2a64 +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-2ac5 +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-2df4 +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-39f5 +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-3f12 +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-4ee8 +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-511e +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-5348 +/home/safari/aolgun/SoftMC_DDR4/sources/xdma_driver/xdma/libxdma.o-.text-61ea diff --git a/sources/xdma_driver/xdma/version.h b/sources/xdma_driver/xdma/version.h new file mode 100644 index 0000000..edd3abf --- /dev/null +++ b/sources/xdma_driver/xdma/version.h @@ -0,0 +1,28 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef __XDMA_VERSION_H__ +#define __XDMA_VERSION_H__ + +#define DRV_MOD_MAJOR 2017 +#define DRV_MOD_MINOR 1 +#define DRV_MOD_PATCHLEVEL 47 + +#define DRV_MODULE_VERSION \ + __stringify(DRV_MOD_MAJOR) "." \ + __stringify(DRV_MOD_MINOR) "." \ + __stringify(DRV_MOD_PATCHLEVEL) + +#define DRV_MOD_VERSION_NUMBER \ + ((DRV_MOD_MAJOR)*1000 + (DRV_MOD_MINOR)*100 + DRV_MOD_PATCHLEVEL) + +#endif /* ifndef __XDMA_VERSION_H__ */ diff --git a/sources/xdma_driver/xdma/xdma_cdev.c b/sources/xdma_driver/xdma/xdma_cdev.c new file mode 100644 index 0000000..b7981f6 --- /dev/null +++ b/sources/xdma_driver/xdma/xdma_cdev.c @@ -0,0 +1,539 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include "xdma_cdev.h" + +struct class *g_xdma_class; + +enum cdev_type { + CHAR_USER, + CHAR_CTRL, + CHAR_XVC, + CHAR_EVENTS, + CHAR_XDMA_H2C, + CHAR_XDMA_C2H, + CHAR_BYPASS_H2C, + CHAR_BYPASS_C2H, + CHAR_BYPASS, +}; + +static const char * const devnode_names[] = { + XDMA_NODE_NAME "%d_user", + XDMA_NODE_NAME "%d_control", + XDMA_NODE_NAME "%d_xvc", + XDMA_NODE_NAME "%d_events_%d", + XDMA_NODE_NAME "%d_h2c_%d", + XDMA_NODE_NAME "%d_c2h_%d", + XDMA_NODE_NAME "%d_bypass_h2c_%d", + XDMA_NODE_NAME "%d_bypass_c2h_%d", + XDMA_NODE_NAME "%d_bypass", +}; + +enum xpdev_flags_bits { + XDF_CDEV_USER, + XDF_CDEV_CTRL, + XDF_CDEV_XVC, + XDF_CDEV_EVENT, + XDF_CDEV_SG, + XDF_CDEV_BYPASS, +}; + +static inline void xpdev_flag_set(struct xdma_pci_dev *xpdev, + enum xpdev_flags_bits fbit) +{ + xpdev->flags |= 1 << fbit; +} + +static inline void xcdev_flag_clear(struct xdma_pci_dev *xpdev, + enum xpdev_flags_bits fbit) +{ + xpdev->flags &= ~(1 << fbit); +} + +static inline int xpdev_flag_test(struct xdma_pci_dev *xpdev, + enum xpdev_flags_bits fbit) +{ + return xpdev->flags & (1 << fbit); +} + +#ifdef __XDMA_SYSFS__ +ssize_t show_device_numbers(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct xdma_pci_dev *xpdev = (struct xdma_pci_dev *)dev_get_drvdata(dev); + + return snprintf(buf, PAGE_SIZE, "%d\t%d\n", + xpdev->major, xpdev->xdev->idx); +} + +static DEVICE_ATTR(xdma_dev_instance, S_IRUGO, show_device_numbers, NULL); +#endif + +static int config_kobject(struct xdma_cdev *xcdev, enum cdev_type type) +{ + int rv = -EINVAL; + struct xdma_dev *xdev = xcdev->xdev; + struct xdma_engine *engine = xcdev->engine; + + switch (type) { + case CHAR_XDMA_H2C: + case CHAR_XDMA_C2H: + case CHAR_BYPASS_H2C: + case CHAR_BYPASS_C2H: + BUG_ON(!engine); + rv = kobject_set_name(&xcdev->cdev.kobj, devnode_names[type], + xdev->idx, engine->channel); + break; + case CHAR_BYPASS: + case CHAR_USER: + case CHAR_CTRL: + case CHAR_XVC: + rv = kobject_set_name(&xcdev->cdev.kobj, devnode_names[type], + xdev->idx); + break; + case CHAR_EVENTS: + rv = kobject_set_name(&xcdev->cdev.kobj, devnode_names[type], + xdev->idx, xcdev->bar); + break; + default: + pr_warn("%s: UNKNOWN type 0x%x.\n", __func__, type); + break; + } + + if (rv) + pr_err("%s: type 0x%x, failed %d.\n", __func__, type, rv); + return rv; +} + +int xcdev_check(const char *fname, struct xdma_cdev *xcdev, bool check_engine) +{ + struct xdma_dev *xdev; + + if (!xcdev || xcdev->magic != MAGIC_CHAR) { + pr_info("%s, xcdev 0x%p, magic 0x%lx.\n", + fname, xcdev, xcdev ? xcdev->magic : 0xFFFFFFFF); + return -EINVAL; + } + + xdev = xcdev->xdev; + if (!xdev || xdev->magic != MAGIC_DEVICE) { + pr_info("%s, xdev 0x%p, magic 0x%lx.\n", + fname, xdev, xdev ? xdev->magic : 0xFFFFFFFF); + return -EINVAL; + } + + if (check_engine) { + struct xdma_engine *engine = xcdev->engine; + if (!engine || engine->magic != MAGIC_ENGINE) { + pr_info("%s, engine 0x%p, magic 0x%lx.\n", fname, + engine, engine ? engine->magic : 0xFFFFFFFF); + return -EINVAL; + } + } + + return 0; +} + +int char_open(struct inode *inode, struct file *file) +{ + struct xdma_cdev *xcdev; + + /* pointer to containing structure of the character device inode */ + xcdev = container_of(inode->i_cdev, struct xdma_cdev, cdev); + BUG_ON(xcdev->magic != MAGIC_CHAR); + /* create a reference to our char device in the opened file */ + file->private_data = xcdev; + + return 0; +} + +/* + * Called when the device goes from used to unused. + */ +int char_close(struct inode *inode, struct file *file) +{ + struct xdma_dev *xdev; + struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data; + + BUG_ON(!xcdev); + BUG_ON(xcdev->magic != MAGIC_CHAR); + + /* fetch device specific data stored earlier during open */ + xdev = xcdev->xdev; + BUG_ON(!xdev); + BUG_ON(xdev->magic != MAGIC_DEVICE); + + return 0; +} + +/* create_xcdev() -- create a character device interface to data or control bus + * + * If at least one SG DMA engine is specified, the character device interface + * is coupled to the SG DMA file operations which operate on the data bus. If + * no engines are specified, the interface is coupled with the control bus. + */ + +static int create_sys_device(struct xdma_cdev *xcdev, enum cdev_type type) +{ + struct xdma_dev *xdev = xcdev->xdev; + struct xdma_engine *engine = xcdev->engine; + int last_param; + + if (type == CHAR_EVENTS) + last_param = xcdev->bar; + else + last_param = engine ? engine->channel : 0; + + xcdev->sys_device = device_create(g_xdma_class, &xdev->pdev->dev, + xcdev->cdevno, NULL, devnode_names[type], xdev->idx, + last_param); + + if (!xcdev->sys_device) { + pr_err("device_create(%s) failed\n", devnode_names[type]); + return -1; + } + + return 0; +} + +static int destroy_xcdev(struct xdma_cdev *cdev) +{ + if (!cdev) { + pr_warn("cdev NULL.\n"); + return 0; + } + if (cdev->magic != MAGIC_CHAR) { + pr_warn("cdev 0x%p magic mismatch 0x%lx\n", cdev, cdev->magic); + return 0; + } + BUG_ON(!cdev->xdev); + BUG_ON(!g_xdma_class); + BUG_ON(!cdev->sys_device); + + if (cdev->sys_device) + device_destroy(g_xdma_class, cdev->cdevno); + + cdev_del(&cdev->cdev); + + return 0; +} + +static int create_xcdev(struct xdma_pci_dev *xpdev, struct xdma_cdev *xcdev, + int bar, struct xdma_engine *engine, + enum cdev_type type) +{ + int rv; + int minor; + struct xdma_dev *xdev = xpdev->xdev; + dev_t dev; + + spin_lock_init(&xcdev->lock); + /* new instance? */ + if (!xpdev->major) { + /* allocate a dynamically allocated char device node */ + int rv = alloc_chrdev_region(&dev, XDMA_MINOR_BASE, + XDMA_MINOR_COUNT, XDMA_NODE_NAME); + + if (rv) { + pr_err("unable to allocate cdev region %d.\n", rv); + return rv; + } + xpdev->major = MAJOR(dev); + } + + /* + * do not register yet, create kobjects and name them, + */ + xcdev->magic = MAGIC_CHAR; + xcdev->cdev.owner = THIS_MODULE; + xcdev->xpdev = xpdev; + xcdev->xdev = xdev; + xcdev->engine = engine; + xcdev->bar = bar; + + rv = config_kobject(xcdev, type); + if (rv < 0) + return rv; + + switch (type) { + case CHAR_USER: + case CHAR_CTRL: + /* minor number is type index for non-SGDMA interfaces */ + minor = type; + cdev_ctrl_init(xcdev); + break; + case CHAR_XVC: + /* minor number is type index for non-SGDMA interfaces */ + minor = type; + cdev_xvc_init(xcdev); + break; + case CHAR_XDMA_H2C: + minor = 32 + engine->channel; + cdev_sgdma_init(xcdev); + break; + case CHAR_XDMA_C2H: + minor = 36 + engine->channel; + cdev_sgdma_init(xcdev); + break; + case CHAR_EVENTS: + minor = 10 + bar; + cdev_event_init(xcdev); + break; + case CHAR_BYPASS_H2C: + minor = 64 + engine->channel; + cdev_bypass_init(xcdev); + break; + case CHAR_BYPASS_C2H: + minor = 68 + engine->channel; + cdev_bypass_init(xcdev); + break; + case CHAR_BYPASS: + minor = 100; + cdev_bypass_init(xcdev); + break; + default: + pr_info("type 0x%x NOT supported.\n", type); + return -EINVAL; + } + xcdev->cdevno = MKDEV(xpdev->major, minor); + + /* bring character device live */ + rv = cdev_add(&xcdev->cdev, xcdev->cdevno, 1); + if (rv < 0) { + pr_err("cdev_add failed %d, type 0x%x.\n", rv, type); + goto unregister_region; + } + + dbg_init("xcdev 0x%p, %u:%u, %s, type 0x%x.\n", + xcdev, xpdev->major, minor, xcdev->cdev.kobj.name, type); + + /* create device on our class */ + if (g_xdma_class) { + rv = create_sys_device(xcdev, type); + if (rv < 0) + goto del_cdev; + } + + return 0; + +del_cdev: + cdev_del(&xcdev->cdev); +unregister_region: + unregister_chrdev_region(dev, XDMA_MINOR_COUNT); + return rv; +} + +void xpdev_destroy_interfaces(struct xdma_pci_dev *xpdev) +{ + int i; + +#ifdef __XDMA_SYSFS__ + device_remove_file(&xpdev->pdev->dev, &dev_attr_xdma_dev_instance); +#endif + + if (xpdev_flag_test(xpdev, XDF_CDEV_SG)) { + /* iterate over channels */ + for (i = 0; i < xpdev->h2c_channel_max; i++) + /* remove SG DMA character device */ + destroy_xcdev(&xpdev->sgdma_h2c_cdev[i]); + for (i = 0; i < xpdev->c2h_channel_max; i++) + destroy_xcdev(&xpdev->sgdma_c2h_cdev[i]); + } + + if (xpdev_flag_test(xpdev, XDF_CDEV_EVENT)) { + for (i = 0; i < xpdev->user_max; i++) + destroy_xcdev(&xpdev->events_cdev[i]); + } + + /* remove control character device */ + if (xpdev_flag_test(xpdev, XDF_CDEV_CTRL)) { + destroy_xcdev(&xpdev->ctrl_cdev); + } + + /* remove user character device */ + if (xpdev_flag_test(xpdev, XDF_CDEV_USER)) { + destroy_xcdev(&xpdev->user_cdev); + } + + if (xpdev_flag_test(xpdev, XDF_CDEV_XVC)) { + destroy_xcdev(&xpdev->xvc_cdev); + } + + if (xpdev_flag_test(xpdev, XDF_CDEV_BYPASS)) { + /* iterate over channels */ + for (i = 0; i < xpdev->h2c_channel_max; i++) + /* remove DMA Bypass character device */ + destroy_xcdev(&xpdev->bypass_h2c_cdev[i]); + for (i = 0; i < xpdev->c2h_channel_max; i++) + destroy_xcdev(&xpdev->bypass_c2h_cdev[i]); + destroy_xcdev(&xpdev->bypass_cdev_base); + } + + if (xpdev->major) + unregister_chrdev_region(MKDEV(xpdev->major, XDMA_MINOR_BASE), XDMA_MINOR_COUNT); +} + +int xpdev_create_interfaces(struct xdma_pci_dev *xpdev) +{ + struct xdma_dev *xdev = xpdev->xdev; + struct xdma_engine *engine; + int i; + int rv = 0; + + /* initialize control character device */ + rv = create_xcdev(xpdev, &xpdev->ctrl_cdev, xdev->config_bar_idx, + NULL, CHAR_CTRL); + if (rv < 0) { + pr_err("create_char(ctrl_cdev) failed\n"); + goto fail; + } + xpdev_flag_set(xpdev, XDF_CDEV_CTRL); + + /* initialize events character device */ + for (i = 0; i < xpdev->user_max; i++) { + rv = create_xcdev(xpdev, &xpdev->events_cdev[i], i, NULL, + CHAR_EVENTS); + if (rv < 0) { + pr_err("create char event %d failed, %d.\n", i, rv); + goto fail; + } + } + xpdev_flag_set(xpdev, XDF_CDEV_EVENT); + + /* iterate over channels */ + for (i = 0; i < xpdev->h2c_channel_max; i++) { + engine = &xdev->engine_h2c[i]; + + if (engine->magic != MAGIC_ENGINE) + continue; + + rv = create_xcdev(xpdev, &xpdev->sgdma_h2c_cdev[i], i, engine, + CHAR_XDMA_H2C); + if (rv < 0) { + pr_err("create char h2c %d failed, %d.\n", i, rv); + goto fail; + } + } + + for (i = 0; i < xpdev->c2h_channel_max; i++) { + engine = &xdev->engine_c2h[i]; + + if (engine->magic != MAGIC_ENGINE) + continue; + + rv = create_xcdev(xpdev, &xpdev->sgdma_c2h_cdev[i], i, engine, + CHAR_XDMA_C2H); + if (rv < 0) { + pr_err("create char c2h %d failed, %d.\n", i, rv); + goto fail; + } + } + xpdev_flag_set(xpdev, XDF_CDEV_SG); + + /* ??? Bypass */ + /* Initialize Bypass Character Device */ + if (xdev->bypass_bar_idx > 0){ + for (i = 0; i < xpdev->h2c_channel_max; i++) { + engine = &xdev->engine_h2c[i]; + + if (engine->magic != MAGIC_ENGINE) + continue; + + rv = create_xcdev(xpdev, &xpdev->bypass_h2c_cdev[i], i, + engine, CHAR_BYPASS_H2C); + if (rv < 0) { + pr_err("create h2c %d bypass I/F failed, %d.\n", + i, rv); + goto fail; + } + } + + for (i = 0; i < xpdev->c2h_channel_max; i++) { + engine = &xdev->engine_c2h[i]; + + if (engine->magic != MAGIC_ENGINE) + continue; + + rv = create_xcdev(xpdev, &xpdev->bypass_c2h_cdev[i], i, + engine, CHAR_BYPASS_C2H); + if (rv < 0) { + pr_err("create c2h %d bypass I/F failed, %d.\n", + i, rv); + goto fail; + } + } + + rv = create_xcdev(xpdev, &xpdev->bypass_cdev_base, + xdev->bypass_bar_idx, NULL, CHAR_BYPASS); + if (rv < 0) { + pr_err("create bypass failed %d.\n", rv); + goto fail; + } + xpdev_flag_set(xpdev, XDF_CDEV_BYPASS); + } + + /* initialize user character device */ + if (xdev->user_bar_idx >= 0) { + rv = create_xcdev(xpdev, &xpdev->user_cdev, xdev->user_bar_idx, + NULL, CHAR_USER); + if (rv < 0) { + pr_err("create_char(user_cdev) failed\n"); + goto fail; + } + xpdev_flag_set(xpdev, XDF_CDEV_USER); + + /* xvc */ + rv = create_xcdev(xpdev, &xpdev->xvc_cdev, xdev->user_bar_idx, + NULL, CHAR_XVC); + if (rv < 0) { + pr_err("create xvc failed, %d.\n", rv); + goto fail; + } + xpdev_flag_set(xpdev, XDF_CDEV_XVC); + } + +#ifdef __XDMA_SYSFS__ + /* sys file */ + rv = device_create_file(&xpdev->pdev->dev, + &dev_attr_xdma_dev_instance); + if (rv) { + pr_err("Failed to create device file \n"); + goto fail; + } +#endif + + return 0; + +fail: + rv = -1; + xpdev_destroy_interfaces(xpdev); + return rv; +} + +int xdma_cdev_init(void) +{ + g_xdma_class = class_create(THIS_MODULE, XDMA_NODE_NAME); + if (IS_ERR(g_xdma_class)) { + dbg_init(XDMA_NODE_NAME ": failed to create class"); + return -1; + } + + return 0; +} + +void xdma_cdev_cleanup(void) +{ + if (g_xdma_class) + class_destroy(g_xdma_class); +} diff --git a/sources/xdma_driver/xdma/xdma_cdev.h b/sources/xdma_driver/xdma/xdma_cdev.h new file mode 100644 index 0000000..9570066 --- /dev/null +++ b/sources/xdma_driver/xdma/xdma_cdev.h @@ -0,0 +1,44 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef __XDMA_CHRDEV_H__ +#define __XDMA_CHRDEV_H__ + +#include <linux/kernel.h> +#include <linux/types.h> +#include <linux/uaccess.h> +#include <linux/errno.h> +#include "xdma_mod.h" + +#define XDMA_NODE_NAME "xdma" +#define XDMA_MINOR_BASE (0) +#define XDMA_MINOR_COUNT (255) + +void xdma_cdev_cleanup(void); +int xdma_cdev_init(void); + +int char_open(struct inode *inode, struct file *file); +int char_close(struct inode *inode, struct file *file); +int xcdev_check(const char *, struct xdma_cdev *, bool); + +void cdev_ctrl_init(struct xdma_cdev *xcdev); +void cdev_xvc_init(struct xdma_cdev *xcdev); +void cdev_event_init(struct xdma_cdev *xcdev); +void cdev_sgdma_init(struct xdma_cdev *xcdev); +void cdev_bypass_init(struct xdma_cdev *xcdev); + +void xpdev_destroy_interfaces(struct xdma_pci_dev *xpdev); +int xpdev_create_interfaces(struct xdma_pci_dev *xpdev); + +int bridge_mmap(struct file *file, struct vm_area_struct *vma); + +#endif /* __XDMA_CHRDEV_H__ */ diff --git a/sources/xdma_driver/xdma/xdma_mod.c b/sources/xdma_driver/xdma/xdma_mod.c new file mode 100644 index 0000000..d6f724a --- /dev/null +++ b/sources/xdma_driver/xdma/xdma_mod.c @@ -0,0 +1,347 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ + +#include <linux/ioctl.h> +#include <linux/types.h> +#include <linux/errno.h> +#include <linux/aer.h> +/* include early, to verify it depends only on the headers above */ +#include "libxdma_api.h" +#include "libxdma.h" +#include "xdma_mod.h" +#include "xdma_cdev.h" +#include "version.h" + +#define DRV_MODULE_NAME "xdma" +#define DRV_MODULE_DESC "Xilinx XDMA Reference Driver" +#define DRV_MODULE_RELDATE "Feb. 2018" + +static char version[] = + DRV_MODULE_DESC " " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n"; + +MODULE_AUTHOR("Xilinx, Inc."); +MODULE_DESCRIPTION(DRV_MODULE_DESC); +MODULE_VERSION(DRV_MODULE_VERSION); +MODULE_LICENSE("Dual BSD/GPL"); + +/* SECTION: Module global variables */ +static int xpdev_cnt = 0; + +static const struct pci_device_id pci_ids[] = { + { PCI_DEVICE(0x10ee, 0x903f), }, + { PCI_DEVICE(0x10ee, 0x9038), }, + { PCI_DEVICE(0x10ee, 0x9028), }, + { PCI_DEVICE(0x10ee, 0x9018), }, + { PCI_DEVICE(0x10ee, 0x9034), }, + { PCI_DEVICE(0x10ee, 0x9024), }, + { PCI_DEVICE(0x10ee, 0x9014), }, + { PCI_DEVICE(0x10ee, 0x9032), }, + { PCI_DEVICE(0x10ee, 0x9022), }, + { PCI_DEVICE(0x10ee, 0x9012), }, + { PCI_DEVICE(0x10ee, 0x9031), }, + { PCI_DEVICE(0x10ee, 0x9021), }, + { PCI_DEVICE(0x10ee, 0x9011), }, + + { PCI_DEVICE(0x10ee, 0x8011), }, + { PCI_DEVICE(0x10ee, 0x8012), }, + { PCI_DEVICE(0x10ee, 0x8014), }, + { PCI_DEVICE(0x10ee, 0x8018), }, + { PCI_DEVICE(0x10ee, 0x8021), }, + { PCI_DEVICE(0x10ee, 0x8022), }, + { PCI_DEVICE(0x10ee, 0x8024), }, + { PCI_DEVICE(0x10ee, 0x8028), }, + { PCI_DEVICE(0x10ee, 0x8031), }, + { PCI_DEVICE(0x10ee, 0x8032), }, + { PCI_DEVICE(0x10ee, 0x8034), }, + { PCI_DEVICE(0x10ee, 0x8038), }, + + { PCI_DEVICE(0x10ee, 0x7011), }, + { PCI_DEVICE(0x10ee, 0x7012), }, + { PCI_DEVICE(0x10ee, 0x7014), }, + { PCI_DEVICE(0x10ee, 0x7018), }, + { PCI_DEVICE(0x10ee, 0x7021), }, + { PCI_DEVICE(0x10ee, 0x7022), }, + { PCI_DEVICE(0x10ee, 0x7024), }, + { PCI_DEVICE(0x10ee, 0x7028), }, + { PCI_DEVICE(0x10ee, 0x7031), }, + { PCI_DEVICE(0x10ee, 0x7032), }, + { PCI_DEVICE(0x10ee, 0x7034), }, + { PCI_DEVICE(0x10ee, 0x7038), }, + + { PCI_DEVICE(0x10ee, 0x6828), }, + { PCI_DEVICE(0x10ee, 0x6830), }, + { PCI_DEVICE(0x10ee, 0x6928), }, + { PCI_DEVICE(0x10ee, 0x6930), }, + { PCI_DEVICE(0x10ee, 0x6A28), }, + { PCI_DEVICE(0x10ee, 0x6A30), }, + { PCI_DEVICE(0x10ee, 0x6D30), }, + + { PCI_DEVICE(0x10ee, 0x4808), }, + { PCI_DEVICE(0x10ee, 0x4828), }, + { PCI_DEVICE(0x10ee, 0x4908), }, + { PCI_DEVICE(0x10ee, 0x4A28), }, + { PCI_DEVICE(0x10ee, 0x4B28), }, + + { PCI_DEVICE(0x10ee, 0x2808), }, + +#ifdef INTERNAL_TESTING + { PCI_DEVICE(0x1d0f, 0x1042), 0}, +#endif + {0,} +}; +MODULE_DEVICE_TABLE(pci, pci_ids); + +static void xpdev_free(struct xdma_pci_dev *xpdev) +{ + struct xdma_dev *xdev = xpdev->xdev; + + pr_info("xpdev 0x%p, destroy_interfaces, xdev 0x%p.\n", xpdev, xdev); + xpdev_destroy_interfaces(xpdev); + xpdev->xdev = NULL; + pr_info("xpdev 0x%p, xdev 0x%p xdma_device_close.\n", xpdev, xdev); + xdma_device_close(xpdev->pdev, xdev); + xpdev_cnt--; + + kfree(xpdev); +} + +static struct xdma_pci_dev *xpdev_alloc(struct pci_dev *pdev) +{ + struct xdma_pci_dev *xpdev = kmalloc(sizeof(*xpdev), GFP_KERNEL); + + if (!xpdev) + return NULL; + memset(xpdev, 0, sizeof(*xpdev)); + + xpdev->magic = MAGIC_DEVICE; + xpdev->pdev = pdev; + xpdev->user_max = MAX_USER_IRQ; + xpdev->h2c_channel_max = XDMA_CHANNEL_NUM_MAX; + xpdev->c2h_channel_max = XDMA_CHANNEL_NUM_MAX; + + xpdev_cnt++; + return xpdev; +} + +static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id) +{ + int rv = 0; + struct xdma_pci_dev *xpdev = NULL; + struct xdma_dev *xdev; + void *hndl; + + xpdev = xpdev_alloc(pdev); + if (!xpdev) + return -ENOMEM; + + hndl = xdma_device_open(DRV_MODULE_NAME, pdev, &xpdev->user_max, + &xpdev->h2c_channel_max, &xpdev->c2h_channel_max); + if (!hndl) + return -EINVAL; + + BUG_ON(xpdev->user_max > MAX_USER_IRQ); + BUG_ON(xpdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX); + BUG_ON(xpdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX); + + if (!xpdev->h2c_channel_max && !xpdev->c2h_channel_max) + pr_warn("NO engine found!\n"); + + if (xpdev->user_max) { + u32 mask = (1 << (xpdev->user_max + 1)) - 1; + + rv = xdma_user_isr_enable(hndl, mask); + if (rv) + goto err_out; + } + + /* make sure no duplicate */ + xdev = xdev_find_by_pdev(pdev); + if (!xdev) { + pr_warn("NO xdev found!\n"); + return -EINVAL; + } + BUG_ON(hndl != xdev ); + + pr_info("%s xdma%d, pdev 0x%p, xdev 0x%p, 0x%p, usr %d, ch %d,%d.\n", + dev_name(&pdev->dev), xdev->idx, pdev, xpdev, xdev, + xpdev->user_max, xpdev->h2c_channel_max, + xpdev->c2h_channel_max); + + xpdev->xdev = hndl; + + rv = xpdev_create_interfaces(xpdev); + if (rv) + goto err_out; + + dev_set_drvdata(&pdev->dev, xpdev); + + return 0; + +err_out: + pr_err("pdev 0x%p, err %d.\n", pdev, rv); + xpdev_free(xpdev); + return rv; +} + +static void remove_one(struct pci_dev *pdev) +{ + struct xdma_pci_dev *xpdev; + + if (!pdev) + return; + + xpdev = dev_get_drvdata(&pdev->dev); + if (!xpdev) + return; + + pr_info("pdev 0x%p, xdev 0x%p, 0x%p.\n", + pdev, xpdev, xpdev->xdev); + xpdev_free(xpdev); + + dev_set_drvdata(&pdev->dev, NULL); +} + +static pci_ers_result_t xdma_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev); + + switch (state) { + case pci_channel_io_normal: + return PCI_ERS_RESULT_CAN_RECOVER; + case pci_channel_io_frozen: + pr_warn("dev 0x%p,0x%p, frozen state error, reset controller\n", + pdev, xpdev); + xdma_device_offline(pdev, xpdev->xdev); + pci_disable_device(pdev); + return PCI_ERS_RESULT_NEED_RESET; + case pci_channel_io_perm_failure: + pr_warn("dev 0x%p,0x%p, failure state error, req. disconnect\n", + pdev, xpdev); + return PCI_ERS_RESULT_DISCONNECT; + } + return PCI_ERS_RESULT_NEED_RESET; +} + +static pci_ers_result_t xdma_slot_reset(struct pci_dev *pdev) +{ + struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev); + + pr_info("0x%p restart after slot reset\n", xpdev); + if (pci_enable_device_mem(pdev)) { + pr_info("0x%p failed to renable after slot reset\n", xpdev); + return PCI_ERS_RESULT_DISCONNECT; + } + + pci_set_master(pdev); + pci_restore_state(pdev); + pci_save_state(pdev); + xdma_device_online(pdev, xpdev->xdev); + + return PCI_ERS_RESULT_RECOVERED; +} + +static void xdma_error_resume(struct pci_dev *pdev) +{ + struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev); + + pr_info("dev 0x%p,0x%p.\n", pdev, xpdev); +#if KERNEL_VERSION(5, 7, 0) <= LINUX_VERSION_CODE + pci_aer_clear_nonfatal_status(pdev); +#else + pci_cleanup_aer_uncorrect_error_status(pdev); +#endif +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,13,0) +static void xdma_reset_prepare(struct pci_dev *pdev) +{ + struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev); + + pr_info("dev 0x%p,0x%p.\n", pdev, xpdev); + xdma_device_offline(pdev, xpdev->xdev); +} + +static void xdma_reset_done(struct pci_dev *pdev) +{ + struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev); + + pr_info("dev 0x%p,0x%p.\n", pdev, xpdev); + xdma_device_online(pdev, xpdev->xdev); +} + +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) +static void xdma_reset_notify(struct pci_dev *pdev, bool prepare) +{ + struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev); + + pr_info("dev 0x%p,0x%p, prepare %d.\n", pdev, xpdev, prepare); + + if (prepare) + xdma_device_offline(pdev, xpdev->xdev); + else + xdma_device_online(pdev, xpdev->xdev); +} +#endif + +static const struct pci_error_handlers xdma_err_handler = { + .error_detected = xdma_error_detected, + .slot_reset = xdma_slot_reset, + .resume = xdma_error_resume, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,13,0) + .reset_prepare = xdma_reset_prepare, + .reset_done = xdma_reset_done, +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0) + .reset_notify = xdma_reset_notify, +#endif +}; + +static struct pci_driver pci_driver = { + .name = DRV_MODULE_NAME, + .id_table = pci_ids, + .probe = probe_one, + .remove = remove_one, + .err_handler = &xdma_err_handler, +}; + +static int __init xdma_mod_init(void) +{ + int rv; + extern unsigned int desc_blen_max; + extern unsigned int sgdma_timeout; + + pr_info("%s", version); + + if (desc_blen_max > XDMA_DESC_BLEN_MAX) + desc_blen_max = XDMA_DESC_BLEN_MAX; + pr_info("desc_blen_max: 0x%x/%u, sgdma_timeout: %u sec.\n", + desc_blen_max, desc_blen_max, sgdma_timeout); + + rv = xdma_cdev_init(); + if (rv < 0) + return rv; + + return pci_register_driver(&pci_driver); +} + +static void __exit xdma_mod_exit(void) +{ + /* unregister this driver from the PCI bus driver */ + dbg_init("pci_unregister_driver.\n"); + pci_unregister_driver(&pci_driver); + xdma_cdev_cleanup(); +} + +module_init(xdma_mod_init); +module_exit(xdma_mod_exit); diff --git a/sources/xdma_driver/xdma/xdma_mod.h b/sources/xdma_driver/xdma/xdma_mod.h new file mode 100644 index 0000000..93202b1 --- /dev/null +++ b/sources/xdma_driver/xdma/xdma_mod.h @@ -0,0 +1,98 @@ +/* + * This file is part of the Xilinx DMA IP Core driver for Linux + * + * Copyright (c) 2016-present, Xilinx, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef __XDMA_MODULE_H__ +#define __XDMA_MODULE_H__ + +#include <linux/types.h> +#include <linux/module.h> +#include <linux/cdev.h> +#include <linux/dma-mapping.h> +#include <linux/delay.h> +#include <linux/fb.h> +#include <linux/fs.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/jiffies.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/mm_types.h> +#include <linux/poll.h> +#include <linux/pci.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/workqueue.h> +#include <linux/aio.h> +#include <linux/splice.h> +#include <linux/version.h> +#include <linux/uio.h> + +#include "libxdma.h" + +#define MAGIC_ENGINE 0xEEEEEEEEUL +#define MAGIC_DEVICE 0xDDDDDDDDUL +#define MAGIC_CHAR 0xCCCCCCCCUL +#define MAGIC_BITSTREAM 0xBBBBBBBBUL + +struct xdma_cdev { + unsigned long magic; /* structure ID for sanity checks */ + struct xdma_pci_dev *xpdev; + struct xdma_dev *xdev; + dev_t cdevno; /* character device major:minor */ + struct cdev cdev; /* character device embedded struct */ + int bar; /* PCIe BAR for HW access, if needed */ + unsigned long base; /* bar access offset */ + struct xdma_engine *engine; /* engine instance, if needed */ + struct xdma_user_irq *user_irq; /* IRQ value, if needed */ + struct device *sys_device; /* sysfs device */ + spinlock_t lock; +}; + +/* XDMA PCIe device specific book-keeping */ +struct xdma_pci_dev { + unsigned long magic; /* structure ID for sanity checks */ + struct pci_dev *pdev; /* pci device struct from probe() */ + struct xdma_dev *xdev; + int major; /* major number */ + int instance; /* instance number */ + int user_max; + int c2h_channel_max; + int h2c_channel_max; + + unsigned int flags; + /* character device structures */ + struct xdma_cdev ctrl_cdev; + struct xdma_cdev sgdma_c2h_cdev[XDMA_CHANNEL_NUM_MAX]; + struct xdma_cdev sgdma_h2c_cdev[XDMA_CHANNEL_NUM_MAX]; + struct xdma_cdev events_cdev[16]; + + struct xdma_cdev user_cdev; + struct xdma_cdev bypass_c2h_cdev[XDMA_CHANNEL_NUM_MAX]; + struct xdma_cdev bypass_h2c_cdev[XDMA_CHANNEL_NUM_MAX]; + struct xdma_cdev bypass_cdev_base; + + struct xdma_cdev xvc_cdev; + + void *data; +}; + +struct xdma_io_cb { + void __user *buf; + size_t len; + unsigned int pages_nr; + struct sg_table sgt; + struct page **pages; +}; + +#endif /* ifndef __XDMA_MODULE_H__ */ |