From 7721980589f4c7fa811689e815a77d63f2556497 Mon Sep 17 00:00:00 2001 From: Ataberk Date: Thu, 2 May 2024 17:22:05 +0200 Subject: Add sources for U50 HBM2 --- sources/api/board.cpp | 20 +++++++++--- sources/api/board.h | 3 +- sources/api/instruction.cpp | 76 +++++++++++++++++++++++++++------------------ sources/api/instruction.h | 41 ++++++++++++------------ sources/api/platform.cpp | 64 +++++++++++++++++++++++++++++++++----- sources/api/platform.h | 5 +++ 6 files changed, 147 insertions(+), 62 deletions(-) (limited to 'sources/api') diff --git a/sources/api/board.cpp b/sources/api/board.cpp index db990ca..d2d2047 100644 --- a/sources/api/board.cpp +++ b/sources/api/board.cpp @@ -6,9 +6,10 @@ #include #include -BoardInterface::BoardInterface(IFACE iface_type) +BoardInterface::BoardInterface(IFACE iface_type, int dimm_select) { this -> iface_type = iface_type; + this -> dimm_select = dimm_select; } BoardInterface::~BoardInterface() { @@ -24,23 +25,31 @@ int BoardInterface::init() { case IFACE::XDMA: { - int fpga_fd = open(TO_FPGA_DEFAULT.c_str(), O_RDWR); + int fpga_fd; + if (dimm_select == 0) + fpga_fd = open(TO_FPGA_DEFAULT.c_str(), O_RDWR); + else + fpga_fd = open("/dev/xdma0_h2c_1", O_RDWR); + if(fpga_fd<0) { std::cerr << "Open to card failed!" << std::endl; return 1; } else - std::cout << "Opened " << TO_FPGA_DEFAULT << " -> " << fpga_fd << std::endl; + std::cout << "Opened " << (dimm_select == 0 ? TO_FPGA_DEFAULT : "/dev/xdma0_h2c_1") << " -> " << fpga_fd << std::endl; to_card = fpga_fd; - fpga_fd = open(FROM_FPGA_DEFAULT.c_str(), O_RDWR); + if (dimm_select == 0) + fpga_fd = open(FROM_FPGA_DEFAULT.c_str(), O_RDWR); + else + fpga_fd = open("/dev/xdma0_c2h_1", O_RDWR); if(fpga_fd<0) { std::cerr << "Open to host failed!" << std::endl; return 1; } else - std::cout << "Opened " << FROM_FPGA_DEFAULT << " -> " << fpga_fd << std::endl; + std::cout << "Opened " << (dimm_select == 0 ? FROM_FPGA_DEFAULT : "/dev/xdma0_c2h_1") << " -> " << fpga_fd << std::endl; from_card = fpga_fd; // allocate page size aligned X page size regions to our buffers if (posix_memalign((void **)&send_buf, 4096 /*alignment */ , SEND_BUF_SIZE + (4096-(SEND_BUF_SIZE % 4096))) != 0) @@ -104,6 +113,7 @@ int BoardInterface::xdma_send(void* data, const uint size) while (count < size) { /* write data to file from memory buffer */ rc = write(fd, buf, size); + // std::cout << rc << std::endl; assert(rc == size || rc == 0); count += rc; } diff --git a/sources/api/board.h b/sources/api/board.h index 0935392..4f08186 100644 --- a/sources/api/board.h +++ b/sources/api/board.h @@ -15,7 +15,7 @@ public: enum class IFACE { XDMA = 0 }; - BoardInterface(IFACE); + BoardInterface(IFACE, int dimm_select = 0); ~BoardInterface(); int init(); int sendData(void* data, const uint size); @@ -25,6 +25,7 @@ private: // XDMA related constructs int to_card; int from_card; + int dimm_select; void* send_buf; void* recv_buf; int xdma_send(void* data, const uint size); diff --git a/sources/api/instruction.cpp b/sources/api/instruction.cpp index 432188f..385f838 100644 --- a/sources/api/instruction.cpp +++ b/sources/api/instruction.cpp @@ -103,7 +103,7 @@ Inst SMC_LDPC(PC_TYPE pc_type, int rt) case PC_TYPE::ACT: pc_reg = 3; break; - case PC_TYPE::ZQ: + case PC_TYPE::SEL_CH: pc_reg = 4; break; case PC_TYPE::REF: @@ -229,76 +229,92 @@ Inst SMC_INFO(int rdcnt) Inst fu_code = (uint64_t)__INFO << __FU_CODE; return op_code | fu_code | (uint64_t) rdcnt; } -Mininst SMC_WRITE(int bar, int ibar, int car, int icar, int BL4, int ap) +Mininst SMC_WRITE(int bar, int ibar, int car, int icar, int rank, int ap) { Mininst fu_code = ((uint64_t)__WRITE) << __DDR_CMD; Mininst i_bar = bar; Mininst i_car = car << __DDR_CAR; Mininst i_ibar = ibar << __DDR_IBAR; Mininst i_icar = icar << __DDR_ICAR; - Mininst i_BL4 = BL4 << __DDR_BL4; + Mininst i_rank = rank << __DDR_RANK; Mininst i_ap = ap <<__DDR_AP; Mininst inst = fu_code | i_bar | i_car | i_ibar | - i_icar | i_BL4 | i_ap; + i_icar | i_rank | i_ap; return inst; } -Mininst SMC_READ(int bar, int ibar, int car, int icar, int BL4, int ap) +Mininst SMC_READ(int bar, int ibar, int car, int icar, int rank, int ap) { Mininst fu_code = ((uint64_t)__READ) << __DDR_CMD; Mininst i_bar = bar; Mininst i_car = car << __DDR_CAR; Mininst i_ibar = ibar << __DDR_IBAR; Mininst i_icar = icar << __DDR_ICAR; - Mininst i_BL4 = BL4 << __DDR_BL4; + Mininst i_rank = rank << __DDR_RANK; Mininst i_ap = ap <<__DDR_AP; Mininst inst = fu_code | i_bar | i_car | i_ibar | - i_icar | i_BL4 | i_ap; + i_icar | i_rank | i_ap; return inst; } -Mininst SMC_PRE(int bar, int ibar, int pall) +Mininst SMC_PRE(int bar, int ibar, int pall, int rank) { Mininst fu_code = ((uint64_t)__PRE) << __DDR_CMD; Mininst i_bar = bar; Mininst i_ibar = ibar << __DDR_IBAR; Mininst i_pall = pall << __DDR_PALL; + Mininst i_rank = rank << __DDR_RANK; - Mininst inst = fu_code | i_bar | i_ibar | i_pall; + + Mininst inst = fu_code | i_bar | i_ibar | i_pall | i_rank; return inst; } -Mininst SMC_ACT(int bar, int ibar, int rar, int irar) +Mininst SMC_ACT(int bar, int ibar, int rar, int irar, int rank) { Mininst fu_code = ((uint64_t)__ACT) << __DDR_CMD; Mininst i_bar = bar; Mininst i_rar = rar << __DDR_RAR; Mininst i_ibar = ibar << __DDR_IBAR; Mininst i_irar = irar << __DDR_IRAR; + Mininst i_rank = rank << __DDR_RANK; + - Mininst inst = fu_code | i_bar | i_rar | i_ibar | i_irar; + Mininst inst = fu_code | i_bar | i_rar | i_ibar | i_irar | i_rank; return inst; } -Mininst SMC_ZQ() + +Mininst SMC_SEL_CH(int channel, int pseudo_channel) { - Mininst fu_code = ((uint64_t)__ZQ) << __DDR_CMD; + Mininst fu_code = ((uint64_t)__SEL_CH) << __DDR_CMD; + Mininst i_rank = pseudo_channel << __DDR_RANK; + Mininst i_channel = channel; - return fu_code; + Mininst inst = fu_code | i_rank | i_channel; + + return inst; } -Mininst SMC_REF() + +Mininst SMC_REF(int rank) { Mininst fu_code = ((uint64_t)__REF) << __DDR_CMD; + Mininst i_rank = rank << __DDR_RANK; + + Mininst inst = fu_code | i_rank; - return fu_code; + return inst; } -Mininst SMC_NOP() +Mininst SMC_NOP(int rank) { Mininst fu_code = ((uint64_t)__NOP) << __DDR_CMD; + Mininst i_rank = rank << __DDR_RANK; - return fu_code; + Mininst inst = fu_code | i_rank; + + return inst; } Inst SMC_SRE() { @@ -455,7 +471,7 @@ void decode_inst(Inst inst) printf("LDPC r%d ACT_COUNTER", rt); break; case 4: - printf("LDPC r%d ZQ_COUNTER", rt); + printf("LDPC r%d SEL_CH_COUNTER", rt); break; case 5: printf("LDPC r%d REF_COUNTER", rt); @@ -538,32 +554,32 @@ void decode_ddr(Mininst i) int irar = (i >> __DDR_IRAR) & 0x1; int pall = (i >> __DDR_PALL) & 0x1; int ap = (i >> __DDR_AP) & 0x1; - int bc = (i >> __DDR_BL4) & 0x1; + int rank = (i >> __DDR_RANK) & 0x1; switch(ddr_code) { case __WRITE: - printf("WR r%d%s r%d%s%s%s", bar, ibar?"++":"", car, icar?"++":"", - ap?" AP":"",bc?" BC":""); + printf("WR r%d%s r%d%s%s %d", bar, ibar?"++":"", car, icar?"++":"", + ap?" AP":"", rank); break; case __READ: - printf("RD r%d%s r%d%s%s%s", bar, ibar?"++":"", car, icar?"++":"", - ap?" AP":"",bc?" BC":""); + printf("RD r%d%s r%d%s%s %d", bar, ibar?"++":"", car, icar?"++":"", + ap?" AP":"", rank); break; case __PRE: - printf("PRE r%d%s%s", bar, ibar?"++":"", pall?" PALL":""); + printf("PRE r%d%s%s %d", bar, ibar?"++":"", pall?" PALL":"", rank); break; case __ACT: - printf("ACT r%d%s r%d%s", bar, ibar?"++":"", rar, irar?"++":""); + printf("ACT r%d%s r%d%s %d", bar, ibar?"++":"", rar, irar?"++":"", rank); break; - case __ZQ: - printf("ZQ"); + case __SEL_CH: + printf("SEL_CH %d", rank); break; case __REF: - printf("REF"); + printf("REF %d", rank); break; case __NOP: - printf("NOP"); + printf("NOP %d", rank); break; } } diff --git a/sources/api/instruction.h b/sources/api/instruction.h index 1be8691..6ec6e61 100644 --- a/sources/api/instruction.h +++ b/sources/api/instruction.h @@ -31,7 +31,7 @@ #define __DDR_IRAR 11 #define __DDR_PALL 11 #define __DDR_AP 9 -#define __DDR_BL4 8 +#define __DDR_RANK 8 // EXE function codes #define __ADD 0 #define __ADDI 1 @@ -55,13 +55,13 @@ #define __LD 0 #define __ST 1 // DDR function codes -#define __WRITE 8 -#define __READ 9 -#define __PRE 10 -#define __ACT 11 -#define __ZQ 12 -#define __REF 13 -#define __NOP 15 +#define __WRITE 8 +#define __READ 9 +#define __PRE 10 +#define __ACT 11 +#define __SEL_CH 12 +#define __REF 13 +#define __NOP 15 // 64 bit instructions typedef uint64_t Inst; @@ -69,7 +69,7 @@ typedef uint64_t Inst; typedef uint16_t Mininst; //Counter types for LDPC Inst -enum class PC_TYPE { WRITE, READ, PRE, ACT, ZQ, REF, CYC }; +enum class PC_TYPE { WRITE, READ, PRE, ACT, SEL_CH, REF, CYC }; /** * Load a word from memory into rt @@ -133,47 +133,50 @@ Inst SMC_SLEEP(uint32_t samt); * @param ibar increment BAR (BAR=BAR+BASR) after issuing the write * @param car column address register ID * @param icar increment CAR (CAR=CAR+CASR) after issuing the write - * @param BL4 burst-chop write + * @param rank target rank * @param ap auto-precharge after issuing write */ -Mininst SMC_WRITE(int bar, int ibar, int car, int icar, int BL4, int ap); +Mininst SMC_WRITE(int bar, int ibar, int car, int icar, int rank, int ap); /** * Generate a DDR-RD command * @param bar bank address register ID * @param ibar increment BAR (BAR=BAR+BASR) after issuing the read * @param car column address register ID * @param icar increment CAR (CAR=CAR+CASR) after issuing the read - * @param BL4 burst-chop read + * @param rank target rank * @param ap auto-precharge after issuing read */ -Mininst SMC_READ(int bar, int ibar, int car, int icar, int BL4, int ap); +Mininst SMC_READ(int bar, int ibar, int car, int icar, int rank, int ap); /** * Generate a DDR-PRE command * @param bar bank address register ID * @param ibar increment BAR (BAR=BAR+BASR) after issuing the write * @param pall precharge all banks + * @param rank target rank */ -Mininst SMC_PRE(int bar, int ibar, int pall); +Mininst SMC_PRE(int bar, int ibar, int pall, int rank = 0); /** * Generate a DDR-RD command * @param bar bank address register ID * @param ibar increment BAR (BAR=BAR+BASR) after issuing the activate * @param rar row address register ID * @param irar increment RAR (RAR=RAR+RASR) after issuing the activate + * @param rank target rank */ -Mininst SMC_ACT(int bar, int ibar, int rar, int irar); +Mininst SMC_ACT(int bar, int ibar, int rar, int irar, int rank = 0); /** - * Generate a zq-calibration command + * Select a channel (for HBM) + * @param pseudo_channel target pseudo channel ID */ -Mininst SMC_ZQ(); +Mininst SMC_SEL_CH(int channel, int pseudo_channel = 0); /** * Generate a refresh command */ -Mininst SMC_REF(); +Mininst SMC_REF(int rank = 0); /** * Generate a no-operation command */ -Mininst SMC_NOP(); +Mininst SMC_NOP(int rank = 0); /** * Enters Self-Refresh Mode */ diff --git a/sources/api/platform.cpp b/sources/api/platform.cpp index 2dae891..27f3bb6 100644 --- a/sources/api/platform.cpp +++ b/sources/api/platform.cpp @@ -22,6 +22,7 @@ SoftMCPlatform::SoftMCPlatform() // 32 KB large read buffer is_dummy = false; xdma_recv_buf = malloc(32*1024); + this->dimm_select = 0; // by default, use DIMM 0 in C3 (or A in the diagram on the spreadsheet) #ifdef PYSMC py_data_buffer = (uint8_t*)malloc(32*1024*sizeof(uint8_t)); @@ -33,6 +34,20 @@ SoftMCPlatform::SoftMCPlatform(bool sandbox) // 32 KB large read buffer is_dummy = sandbox; xdma_recv_buf = malloc(32*1024); + this->dimm_select = 0; +} + +SoftMCPlatform::SoftMCPlatform(int dimm_select) +{ + // 32 KB large read buffer + is_dummy = false; + xdma_recv_buf = malloc(32*1024); + + this->dimm_select = dimm_select; + + #ifdef PYSMC + py_data_buffer = (uint8_t*)malloc(32*1024*sizeof(uint8_t)); + #endif } SoftMCPlatform::~SoftMCPlatform(){ @@ -66,11 +81,14 @@ int SoftMCPlatform::init(){ instr_buf = malloc(INSTR_BUF_SIZE); memset(instr_buf, 0, INSTR_BUF_SIZE); - iface = new BoardInterface(BoardInterface::IFACE::XDMA); - if(!iface -> init()) - return SOFTMC_SUCCESS; - else + iface = new BoardInterface(BoardInterface::IFACE::XDMA, this->dimm_select); + + int init = iface -> init(); + + if (init) return SOFTMC_ERR; + + return SOFTMC_SUCCESS; } } @@ -120,6 +138,7 @@ void SoftMCPlatform::execute(Program &prog) receiver.join(); receiver = std::thread(&SoftMCPlatform::consumeData, this); + int sent = iface -> sendData(instr_buf, bytes*4 /*in bytes*/); memset(instr_buf, 0, bytes*4); free(iseq); @@ -143,12 +162,13 @@ void SoftMCPlatform::consumeData() xdma_read_size = recvd-32; xdma_read_size /= 4; int total_size = xdma_read_size; + int pushsz = api_recv_buf.push(((int*) xdma_recv_buf), xdma_read_size); while(pushsz < total_size) pushsz += api_recv_buf.push(((int*) xdma_recv_buf) + pushsz, xdma_read_size = (total_size - pushsz)); -// printf("SoftMCPlatform::consumedata(): spsc filled with %d beats of data\n", pushsz); -// printf("Consume total: %d\n",++consume_total); + // printf("SoftMCPlatform::consumedata(): spsc filled with %d beats of data\n", pushsz); + // printf("Consume total: %d\n",++consume_total); assert(pushsz == total_size && "Unexpected amount of data pushed into spsc\n"); @@ -272,6 +292,36 @@ void SoftMCPlatform::set_aref(const bool on) } } +void SoftMCPlatform::read_HBM_temperature() +{ + ((uint8_t*) instr_buf)[8] = (uint8_t) 0x10; + int sent = iface -> sendData(instr_buf, 32 /*in bytes*/); + // We do not need to zero out the whole buffer + memset(instr_buf, 0, 512); + + sent = iface -> sendData(instr_buf, 512 /*in bytes*/); + + int recvd = iface -> recvData((void*)xdma_recv_buf, 32*1024); + + printf("Temp1: %d Celsius\n", ((uint8_t*) xdma_recv_buf)[32]); + printf("Temp2: %d Celsius\n", ((uint8_t*) xdma_recv_buf)[33]); +} + +int SoftMCPlatform::return_HBM_temperature() +{ + ((uint8_t*) instr_buf)[8] = (uint8_t) 0x10; + int sent = iface -> sendData(instr_buf, 32 /*in bytes*/); + // We do not need to zero out the whole buffer + memset(instr_buf, 0, 512); + + sent = iface -> sendData(instr_buf, 512 /*in bytes*/); + + int recvd = iface -> recvData((void*)xdma_recv_buf, 32*1024); + + // Return stack 2's temperature + return (int)(((uint8_t*) xdma_recv_buf)[33]); +} + void SoftMCPlatform::readRegisterDump() { if(is_dummy) @@ -293,4 +343,4 @@ void SoftMCPlatform::readRegisterDump() printf("%x", ((uint32_t*)readData)[r]); printf("\n"); } -} \ No newline at end of file +} diff --git a/sources/api/platform.h b/sources/api/platform.h index 277dcdf..f79e319 100644 --- a/sources/api/platform.h +++ b/sources/api/platform.h @@ -21,6 +21,7 @@ class SoftMCPlatform{ public: SoftMCPlatform(); SoftMCPlatform(bool); + SoftMCPlatform(int dimm_select); ~SoftMCPlatform(); /** * Initializes the whole platform @@ -60,6 +61,9 @@ class SoftMCPlatform{ */ void set_aref(bool on); + void read_HBM_temperature(); + int return_HBM_temperature(); + /** * Used along with Program::dumpRegisters to read register content */ @@ -67,6 +71,7 @@ class SoftMCPlatform{ private: bool is_dummy; + int dimm_select; BoardInterface *iface; void* instr_buf; -- cgit v1.2.3