Add sources for U50 HBM2

19 files changed, 682 insertions, 155 deletions

diff --git a/sources/api/board.cpp b/sources/api/board.cpp
index db990ca..d2d2047 100644
--- a/sources/api/board.cpp
+++ b/sources/api/board.cpp
@@ -6,9 +6,10 @@
 #include <fcntl.h>
 #include <string.h>
 
-BoardInterface::BoardInterface(IFACE iface_type)
+BoardInterface::BoardInterface(IFACE iface_type, int dimm_select)
 {
   this -> iface_type = iface_type;
+  this -> dimm_select = dimm_select;
 }
 BoardInterface::~BoardInterface()
 {
@@ -24,23 +25,31 @@ int BoardInterface::init()
   {
     case IFACE::XDMA:
     {
-      int fpga_fd = open(TO_FPGA_DEFAULT.c_str(), O_RDWR);
+      int fpga_fd;
+      if (dimm_select == 0)
+        fpga_fd = open(TO_FPGA_DEFAULT.c_str(), O_RDWR);
+      else
+        fpga_fd = open("/dev/xdma0_h2c_1", O_RDWR);
+  
       if(fpga_fd<0)
       {
         std::cerr << "Open to card failed!" << std::endl;
         return 1;
       }
       else
-        std::cout << "Opened " << TO_FPGA_DEFAULT <<  " -> " << fpga_fd << std::endl;
+        std::cout << "Opened " << (dimm_select == 0 ? TO_FPGA_DEFAULT : "/dev/xdma0_h2c_1") <<  " -> " << fpga_fd << std::endl;
       to_card = fpga_fd;
-      fpga_fd = open(FROM_FPGA_DEFAULT.c_str(), O_RDWR);
+      if (dimm_select == 0)
+        fpga_fd = open(FROM_FPGA_DEFAULT.c_str(), O_RDWR);
+      else
+        fpga_fd = open("/dev/xdma0_c2h_1", O_RDWR);
       if(fpga_fd<0)
       {
         std::cerr << "Open to host failed!" << std::endl;
         return 1;
       }
       else
-        std::cout << "Opened " << FROM_FPGA_DEFAULT <<  " -> " << fpga_fd << std::endl;
+        std::cout << "Opened " << (dimm_select == 0 ? FROM_FPGA_DEFAULT : "/dev/xdma0_c2h_1") <<  " -> " << fpga_fd << std::endl;
       from_card = fpga_fd;
       // allocate page size aligned X page size regions to our buffers
       if (posix_memalign((void **)&send_buf, 4096 /*alignment */ , SEND_BUF_SIZE + (4096-(SEND_BUF_SIZE % 4096))) != 0)
@@ -104,6 +113,7 @@ int BoardInterface::xdma_send(void* data, const uint size)
   while (count < size) {
     /* write data to file from memory buffer */
     rc = write(fd, buf, size);
+    // std::cout << rc << std::endl;
     assert(rc == size || rc == 0);
     count += rc;
   }
diff --git a/sources/api/board.h b/sources/api/board.h
index 0935392..4f08186 100644
--- a/sources/api/board.h
+++ b/sources/api/board.h
@@ -15,7 +15,7 @@ public:
   enum class IFACE {
       XDMA = 0
   };
-  BoardInterface(IFACE);
+  BoardInterface(IFACE, int dimm_select = 0);
   ~BoardInterface();
   int init();
   int sendData(void* data, const uint size);
@@ -25,6 +25,7 @@ private:
   // XDMA related constructs
   int to_card;
   int from_card;
+  int dimm_select;
   void* send_buf;
   void* recv_buf;
   int xdma_send(void* data, const uint size);
diff --git a/sources/api/instruction.cpp b/sources/api/instruction.cpp
index 432188f..385f838 100644
--- a/sources/api/instruction.cpp
+++ b/sources/api/instruction.cpp
@@ -103,7 +103,7 @@ Inst SMC_LDPC(PC_TYPE pc_type, int rt)
     case PC_TYPE::ACT:
       pc_reg = 3;
       break;
-    case PC_TYPE::ZQ:
+    case PC_TYPE::SEL_CH:
       pc_reg = 4;
       break;
     case PC_TYPE::REF:
@@ -229,76 +229,92 @@ Inst SMC_INFO(int rdcnt)
   Inst fu_code = (uint64_t)__INFO << __FU_CODE;
   return op_code | fu_code | (uint64_t) rdcnt;
 }
-Mininst SMC_WRITE(int bar, int ibar, int car, int icar, int BL4, int ap)
+Mininst SMC_WRITE(int bar, int ibar, int car, int icar, int rank, int ap)
 {
   Mininst fu_code   = ((uint64_t)__WRITE) << __DDR_CMD;
   Mininst i_bar     = bar;
   Mininst i_car     = car << __DDR_CAR;
   Mininst i_ibar    = ibar << __DDR_IBAR;
   Mininst i_icar    = icar << __DDR_ICAR;
-  Mininst i_BL4     = BL4 << __DDR_BL4;
+  Mininst i_rank    = rank << __DDR_RANK;
   Mininst i_ap      = ap <<__DDR_AP;
 
   Mininst inst = fu_code | i_bar | i_car | i_ibar |
-          i_icar | i_BL4 | i_ap;
+          i_icar | i_rank | i_ap;
 
   return inst;
 }
-Mininst SMC_READ(int bar, int ibar, int car, int icar, int BL4, int ap)
+Mininst SMC_READ(int bar, int ibar, int car, int icar, int rank, int ap)
 {
   Mininst fu_code   = ((uint64_t)__READ) << __DDR_CMD;
   Mininst i_bar     = bar;
   Mininst i_car     = car << __DDR_CAR;
   Mininst i_ibar    = ibar << __DDR_IBAR;
   Mininst i_icar    = icar << __DDR_ICAR;
-  Mininst i_BL4     = BL4 << __DDR_BL4;
+  Mininst i_rank    = rank << __DDR_RANK;
   Mininst i_ap      = ap <<__DDR_AP;
 
   Mininst inst = fu_code | i_bar | i_car | i_ibar |
-          i_icar | i_BL4 | i_ap;
+          i_icar | i_rank | i_ap;
 
   return inst;
 }
-Mininst SMC_PRE(int bar, int ibar, int pall)
+Mininst SMC_PRE(int bar, int ibar, int pall, int rank)
 {
   Mininst fu_code   = ((uint64_t)__PRE) << __DDR_CMD;
   Mininst i_bar     = bar;
   Mininst i_ibar    = ibar << __DDR_IBAR;
   Mininst i_pall    = pall << __DDR_PALL;
+  Mininst i_rank    = rank << __DDR_RANK;
 
-  Mininst inst = fu_code | i_bar | i_ibar | i_pall;
+
+  Mininst inst = fu_code | i_bar | i_ibar | i_pall | i_rank;
 
   return inst;
 }
-Mininst SMC_ACT(int bar, int ibar, int rar, int irar)
+Mininst SMC_ACT(int bar, int ibar, int rar, int irar, int rank)
 {
   Mininst fu_code   = ((uint64_t)__ACT) << __DDR_CMD;
   Mininst i_bar     = bar;
   Mininst i_rar     = rar << __DDR_RAR;
   Mininst i_ibar    = ibar << __DDR_IBAR;
   Mininst i_irar    = irar << __DDR_IRAR;
+  Mininst i_rank    = rank << __DDR_RANK;
+
 
-  Mininst inst = fu_code | i_bar | i_rar | i_ibar | i_irar;
+  Mininst inst = fu_code | i_bar | i_rar | i_ibar | i_irar | i_rank;
 
   return inst;
 }
-Mininst SMC_ZQ()
+
+Mininst SMC_SEL_CH(int channel, int pseudo_channel)
 {
-  Mininst fu_code   = ((uint64_t)__ZQ) << __DDR_CMD;
+  Mininst fu_code   = ((uint64_t)__SEL_CH) << __DDR_CMD;
+  Mininst i_rank    = pseudo_channel << __DDR_RANK;
+  Mininst i_channel = channel;
 
-  return fu_code;
+  Mininst inst = fu_code | i_rank | i_channel;
+
+  return inst;
 }
-Mininst SMC_REF()
+
+Mininst SMC_REF(int rank)
 {
   Mininst fu_code   = ((uint64_t)__REF) << __DDR_CMD;
+  Mininst i_rank    = rank << __DDR_RANK;
+
+  Mininst inst = fu_code | i_rank; 
 
-  return fu_code;
+  return inst;
 }
-Mininst SMC_NOP()
+Mininst SMC_NOP(int rank)
 {
   Mininst fu_code   = ((uint64_t)__NOP) << __DDR_CMD;
+  Mininst i_rank    = rank << __DDR_RANK;
 
-  return fu_code;
+  Mininst inst = fu_code | i_rank;
+  
+  return inst;
 }
 Inst SMC_SRE()
 {
@@ -455,7 +471,7 @@ void decode_inst(Inst inst)
               printf("LDPC r%d ACT_COUNTER", rt);
               break;
             case 4:
-              printf("LDPC r%d ZQ_COUNTER", rt);
+              printf("LDPC r%d SEL_CH_COUNTER", rt);
               break;
             case 5:
               printf("LDPC r%d REF_COUNTER", rt);
@@ -538,32 +554,32 @@ void decode_ddr(Mininst i)
   int irar     = (i >> __DDR_IRAR) & 0x1;
   int pall     = (i >> __DDR_PALL) & 0x1;
   int ap       = (i >> __DDR_AP) & 0x1;
-  int bc       = (i >> __DDR_BL4) & 0x1;
+  int rank     = (i >> __DDR_RANK) & 0x1;
 
   switch(ddr_code)
   {
     case __WRITE:
-      printf("WR r%d%s r%d%s%s%s", bar, ibar?"++":"", car, icar?"++":"",
-        ap?" AP":"",bc?" BC":"");
+      printf("WR r%d%s r%d%s%s %d", bar, ibar?"++":"", car, icar?"++":"",
+        ap?" AP":"", rank);
       break;
     case __READ:
-      printf("RD r%d%s r%d%s%s%s", bar, ibar?"++":"", car, icar?"++":"",
-        ap?" AP":"",bc?" BC":"");
+      printf("RD r%d%s r%d%s%s %d", bar, ibar?"++":"", car, icar?"++":"",
+        ap?" AP":"", rank);
       break;
     case __PRE:
-      printf("PRE r%d%s%s", bar, ibar?"++":"", pall?" PALL":"");
+      printf("PRE r%d%s%s %d", bar, ibar?"++":"", pall?" PALL":"", rank);
       break;
     case __ACT:
-      printf("ACT r%d%s r%d%s", bar, ibar?"++":"", rar, irar?"++":"");
+      printf("ACT r%d%s r%d%s %d", bar, ibar?"++":"", rar, irar?"++":"", rank);
       break;
-    case __ZQ:
-      printf("ZQ");
+    case __SEL_CH:
+      printf("SEL_CH %d", rank);
       break;
     case __REF:
-      printf("REF");
+      printf("REF %d", rank);
       break;
     case __NOP:
-      printf("NOP");
+      printf("NOP %d", rank);
       break;
   }
 }
diff --git a/sources/api/instruction.h b/sources/api/instruction.h
index 1be8691..6ec6e61 100644
--- a/sources/api/instruction.h
+++ b/sources/api/instruction.h
@@ -31,7 +31,7 @@
 #define __DDR_IRAR  11
 #define __DDR_PALL  11
 #define __DDR_AP    9
-#define __DDR_BL4   8
+#define __DDR_RANK  8
 // EXE function codes
 #define __ADD   0
 #define __ADDI  1
@@ -55,13 +55,13 @@
 #define __LD    0
 #define __ST    1
 // DDR function codes
-#define __WRITE 8
-#define __READ  9
-#define __PRE   10
-#define __ACT   11
-#define __ZQ    12
-#define __REF   13
-#define __NOP   15
+#define __WRITE   8
+#define __READ    9
+#define __PRE     10
+#define __ACT     11
+#define __SEL_CH  12
+#define __REF     13
+#define __NOP     15
 
 // 64 bit instructions
 typedef uint64_t Inst;
@@ -69,7 +69,7 @@ typedef uint64_t Inst;
 typedef uint16_t Mininst;
 
 //Counter types for LDPC Inst
-enum class PC_TYPE { WRITE, READ, PRE, ACT, ZQ, REF, CYC };
+enum class PC_TYPE { WRITE, READ, PRE, ACT, SEL_CH, REF, CYC };
 
 /**
  * Load a word from memory into rt
@@ -133,47 +133,50 @@ Inst SMC_SLEEP(uint32_t samt);
  * @param ibar increment BAR (BAR=BAR+BASR) after issuing the write
  * @param car column address register ID
  * @param icar increment CAR (CAR=CAR+CASR) after issuing the write
- * @param BL4 burst-chop write
+ * @param rank target rank
  * @param ap auto-precharge after issuing write
  */
-Mininst SMC_WRITE(int bar, int ibar, int car, int icar, int BL4, int ap);
+Mininst SMC_WRITE(int bar, int ibar, int car, int icar, int rank, int ap);
 /**
  * Generate a DDR-RD command
  * @param bar bank address register ID
  * @param ibar increment BAR (BAR=BAR+BASR) after issuing the read
  * @param car column address register ID
  * @param icar increment CAR (CAR=CAR+CASR) after issuing the read
- * @param BL4 burst-chop read
+ * @param rank target rank
  * @param ap auto-precharge after issuing read
  */
-Mininst SMC_READ(int bar, int ibar, int car, int icar, int BL4, int ap);
+Mininst SMC_READ(int bar, int ibar, int car, int icar, int rank, int ap);
 /**
  * Generate a DDR-PRE command
  * @param bar bank address register ID
  * @param ibar increment BAR (BAR=BAR+BASR) after issuing the write
  * @param pall precharge all banks
+ * @param rank target rank
  */
-Mininst SMC_PRE(int bar, int ibar, int pall);
+Mininst SMC_PRE(int bar, int ibar, int pall, int rank = 0);
 /**
  * Generate a DDR-RD command
  * @param bar bank address register ID
  * @param ibar increment BAR (BAR=BAR+BASR) after issuing the activate
  * @param rar row address register ID
  * @param irar increment RAR (RAR=RAR+RASR) after issuing the activate
+ * @param rank target rank
  */
-Mininst SMC_ACT(int bar, int ibar, int rar, int irar);
+Mininst SMC_ACT(int bar, int ibar, int rar, int irar, int rank = 0);
 /**
- * Generate a zq-calibration command
+ * Select a channel (for HBM)
+ * @param pseudo_channel target pseudo channel ID
  */ 
-Mininst SMC_ZQ();
+Mininst SMC_SEL_CH(int channel, int pseudo_channel = 0);
 /**
  * Generate a refresh command
  */ 
-Mininst SMC_REF();
+Mininst SMC_REF(int rank = 0);
 /**
  * Generate a no-operation command
  */ 
-Mininst SMC_NOP();
+Mininst SMC_NOP(int rank = 0);
 /**
  * Enters Self-Refresh Mode
  */
diff --git a/sources/api/platform.cpp b/sources/api/platform.cpp
index 2dae891..27f3bb6 100644
--- a/sources/api/platform.cpp
+++ b/sources/api/platform.cpp
@@ -22,6 +22,7 @@ SoftMCPlatform::SoftMCPlatform()
   // 32 KB large read buffer
   is_dummy = false;
   xdma_recv_buf = malloc(32*1024);
+  this->dimm_select = 0; // by default, use DIMM 0 in C3 (or A in the diagram on the spreadsheet)
 
   #ifdef PYSMC
   py_data_buffer = (uint8_t*)malloc(32*1024*sizeof(uint8_t));
@@ -33,6 +34,20 @@ SoftMCPlatform::SoftMCPlatform(bool sandbox)
   // 32 KB large read buffer
   is_dummy = sandbox;
   xdma_recv_buf = malloc(32*1024);
+  this->dimm_select = 0;
+}
+
+SoftMCPlatform::SoftMCPlatform(int dimm_select)
+{
+  // 32 KB large read buffer
+  is_dummy = false;
+  xdma_recv_buf = malloc(32*1024);
+
+  this->dimm_select = dimm_select;
+
+  #ifdef PYSMC
+  py_data_buffer = (uint8_t*)malloc(32*1024*sizeof(uint8_t));
+  #endif
 }
 
 SoftMCPlatform::~SoftMCPlatform(){
@@ -66,11 +81,14 @@ int SoftMCPlatform::init(){
     instr_buf = malloc(INSTR_BUF_SIZE);
     memset(instr_buf, 0, INSTR_BUF_SIZE);
 
-    iface = new BoardInterface(BoardInterface::IFACE::XDMA);
-    if(!iface -> init())
-      return SOFTMC_SUCCESS;
-    else
+    iface = new BoardInterface(BoardInterface::IFACE::XDMA, this->dimm_select);
+  
+    int init = iface -> init();
+
+    if (init)
       return SOFTMC_ERR;
+
+    return SOFTMC_SUCCESS;
   }
 }
 
@@ -120,6 +138,7 @@ void SoftMCPlatform::execute(Program &prog)
       receiver.join();
 
     receiver = std::thread(&SoftMCPlatform::consumeData, this);
+
     int sent = iface -> sendData(instr_buf, bytes*4 /*in bytes*/);
     memset(instr_buf, 0, bytes*4);
     free(iseq);
@@ -143,12 +162,13 @@ void SoftMCPlatform::consumeData()
       xdma_read_size = recvd-32;
     xdma_read_size /= 4;
     int total_size = xdma_read_size;
+
     int pushsz = api_recv_buf.push(((int*) xdma_recv_buf), xdma_read_size);
     while(pushsz < total_size)
       pushsz += api_recv_buf.push(((int*) xdma_recv_buf) + pushsz, xdma_read_size = (total_size - pushsz));
 
-//    printf("SoftMCPlatform::consumedata(): spsc filled with %d beats of data\n", pushsz);
-//    printf("Consume total: %d\n",++consume_total);
+  //  printf("SoftMCPlatform::consumedata(): spsc filled with %d beats of data\n", pushsz);
+  //  printf("Consume total: %d\n",++consume_total);
 
     assert(pushsz == total_size &&
       "Unexpected amount of data pushed into spsc\n");
@@ -272,6 +292,36 @@ void SoftMCPlatform::set_aref(const bool on)
   }
 }
 
+void SoftMCPlatform::read_HBM_temperature()
+{
+    ((uint8_t*) instr_buf)[8] = (uint8_t) 0x10;
+    int sent = iface -> sendData(instr_buf, 32 /*in bytes*/);
+    // We do not need to zero out the whole buffer
+    memset(instr_buf, 0, 512);
+
+    sent = iface -> sendData(instr_buf, 512 /*in bytes*/);
+
+    int recvd = iface -> recvData((void*)xdma_recv_buf, 32*1024);
+
+    printf("Temp1: %d Celsius\n", ((uint8_t*) xdma_recv_buf)[32]);
+    printf("Temp2: %d Celsius\n", ((uint8_t*) xdma_recv_buf)[33]);
+}
+
+int SoftMCPlatform::return_HBM_temperature()
+{
+    ((uint8_t*) instr_buf)[8] = (uint8_t) 0x10;
+    int sent = iface -> sendData(instr_buf, 32 /*in bytes*/);
+    // We do not need to zero out the whole buffer
+    memset(instr_buf, 0, 512);
+
+    sent = iface -> sendData(instr_buf, 512 /*in bytes*/);
+
+    int recvd = iface -> recvData((void*)xdma_recv_buf, 32*1024);
+
+    // Return stack 2's temperature
+    return (int)(((uint8_t*) xdma_recv_buf)[33]);
+}
+
 void SoftMCPlatform::readRegisterDump()
 {
   if(is_dummy)
@@ -293,4 +343,4 @@ void SoftMCPlatform::readRegisterDump()
     printf("%x", ((uint32_t*)readData)[r]);
     printf("\n");
   }
-}
\ No newline at end of file
+}
diff --git a/sources/api/platform.h b/sources/api/platform.h
index 277dcdf..f79e319 100644
--- a/sources/api/platform.h
+++ b/sources/api/platform.h
@@ -21,6 +21,7 @@ class SoftMCPlatform{
   public:
     SoftMCPlatform();
     SoftMCPlatform(bool);
+    SoftMCPlatform(int dimm_select);
     ~SoftMCPlatform();
     /**
      * Initializes the whole platform
@@ -60,6 +61,9 @@ class SoftMCPlatform{
      */
     void set_aref(bool on);
 
+    void read_HBM_temperature();
+    int return_HBM_temperature();
+
     /**
      * Used along with Program::dumpRegisters to read register content
      */
@@ -67,6 +71,7 @@ class SoftMCPlatform{
 
   private:
     bool is_dummy;
+    int dimm_select;
 
     BoardInterface *iface;
     void* instr_buf;
diff --git a/sources/apps/Smalltest_HBM/.gitignore b/sources/apps/Smalltest_HBM/.gitignore
new file mode 100644
index 0000000..57855ab
--- /dev/null
+++ b/sources/apps/Smalltest_HBM/.gitignore
@@ -0,0 +1 @@
+SoftMC_rdwr
diff --git a/sources/apps/Smalltest_HBM/Makefile b/sources/apps/Smalltest_HBM/Makefile
new file mode 100755
index 0000000..c7498df
--- /dev/null
+++ b/sources/apps/Smalltest_HBM/Makefile
@@ -0,0 +1,32 @@
+program_NAME := SoftMC_rdwr
+program_CXX_SRCS := read_write.cpp $(wildcard ../../api/*.c) $(wildcard ../../api/*.cpp)
+program_CXX_OBJS := ${program_CXX_SRCS:.cpp=.o}
+program_CXX_OBJS := ${program_CXX_OBJS:.c=.o}
+program_OBJS := $(program_CXX_OBJS)
+program_INCLUDE_DIRS := ../../api ../../../boost-lib
+program_LIBRARY_DIRS :=
+program_LIBRARIES := 
+CPPFLAGS += -g -std=c++11 -pthread -O3
+
+CPPFLAGS += $(foreach includedir,$(program_INCLUDE_DIRS),-I$(includedir))
+LDFLAGS += $(foreach librarydir,$(program_LIBRARY_DIRS),-L$(librarydir))
+LDFLAGS += $(foreach library,$(program_LIBRARIES),-l$(library))
+
+CC=g++
+
+.PHONY: all clean distclean
+
+all: $(program_NAME)
+
+$(program_NAME): $(program_OBJS)
+	$(CC) $(CPPFLAGS) $(program_OBJS) -o $(program_NAME) $(LDFLAGS)
+
+clean:
+	@- $(RM) $(program_NAME)
+	@- $(RM) $(program_OBJS)
+
+parser: 
+	$(MAKE) -C ../../api/lexyacc
+	cp ../../api/lexyacc/smc_parser .
+
+distclean: clean
diff --git a/sources/apps/Smalltest_HBM/read_write.cpp b/sources/apps/Smalltest_HBM/read_write.cpp
new file mode 100644
index 0000000..88116a6
--- /dev/null
+++ b/sources/apps/Smalltest_HBM/read_write.cpp
@@ -0,0 +1,304 @@
+#include "instruction.h"
+#include "prog.h"
+#include "platform.h"
+#include <fstream>
+#include <iostream>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <cstring>
+#include <list>
+
+// A 4 GB per stack device has 4 Gb per channel, and each channel has
+// two 2 Gb or 256 MB pseudo channels.
+
+// THIS LOOKS LIKE THE CORRECT ORGANIZATION:
+// 32 byte per column x 32 (cols) x 16K (rows) x 16 (banks) x 2 PC x 8 channels
+
+using namespace std;
+
+#define USE_SMC_FILE 0
+
+#define PC_UNDER_TEST 1
+
+#define NUM_CH 8
+#define NUM_PC 2
+#define NUM_BANKS 16
+#define NUM_ROWS 1024*16
+#define NUM_COLS 32
+#define BYTES_PER_READ 32
+#define ROW_SIZE (NUM_COLS*BYTES_PER_READ)
+
+#define NUM_ROWS_REG  8
+#define NUM_BANKS_REG 11
+// Stride register ids are fixed and should not be changed
+// CASR should always be reg 0
+// BASR should always be reg 1
+// RASR should always be reg 2
+#define CASR 0
+#define BASR 1
+#define RASR 2
+
+#define PATTERN_REG 12
+
+#define BAR 7
+#define RAR 6
+#define CAR 4
+
+#define NUM_COLS_REG 14
+#define LOOP_COLS 13
+
+#define NUM_CH_REG 3
+#define LOOP_CH 5
+
+#define TEMP_PATTERN_REG 15
+
+/**
+ * @return an instruction formed by NOPs
+ */
+Inst all_nops()
+{
+  return  __pack_mininsts(SMC_NOP(), SMC_NOP(), SMC_NOP(), SMC_NOP());
+}
+
+int main()
+{
+  SoftMCPlatform platform(false);
+  int err;
+  uint32_t wr_pattern;
+  // buffer allocated for reading data from the board
+  uint8_t buf[ROW_SIZE * 2]; // * 2 because of PCs
+
+  // Initialize the platform, opens file descriptors for the board PCI-E interface.
+  if((err = platform.init()) != SOFTMC_SUCCESS){
+      cerr << "Could not initialize SoftMC Platform: " << err << endl;
+  }
+  // reset the board to hopefully restore the board's state
+  platform.reset_fpga();
+
+  Program program;
+  /* SoftMC programs are formed sequentially by adding
+    * instructions to the program one by one. Instructions
+    * can be added to the program using add_inst().
+    */
+
+  program.add_inst(SMC_LI(NUM_ROWS, NUM_ROWS_REG)); // load NUM_ROWS into NUM_ROWS_REG
+  program.add_inst(SMC_LI(NUM_BANKS, NUM_BANKS_REG)); // load NUM_BANKS into NUM_BANKS_REG
+  program.add_inst(SMC_LI(NUM_CH, NUM_CH_REG)); // load NUM_CH into NUM_CH_REG
+  
+  // Stride registers are used to increment the row,bank,column registers without
+  // using regular (non-DDR) instructions. This way the DRAM array can be traversed much faster
+
+  program.add_inst(SMC_LI(1, CASR)); // Load 1 into CASR since each READ reads 1 column but also skip one because it is another PC
+  program.add_inst(SMC_LI(1, BASR)); // Load 1 into BASR
+  program.add_inst(SMC_LI(1, RASR)); // Load 1 into RASR
+
+  // Initialize PRNG, this test will randomly generate
+  // the first data to write into DRAM, it will modify
+  // the data by cyclic shifting and multiplying it with itself
+  // during runtime (this will be further explained down below).
+  srand(time(NULL)+getpid());
+  uint16_t rand1 = (uint16_t) rand();
+  wr_pattern = rand1 << 16;
+  wr_pattern |= ((uint16_t)rand());
+  printf("wr_pattern: %x\n", wr_pattern);
+
+  program.add_inst(SMC_LI(0, LOOP_CH)); // load NUM_CH into NUM_CH_REG
+
+  program.add_label("CH_BEGIN");
+
+  // Load WR data into pattern register whose content
+  // will be replicated among all words in the wide register
+  // Wide register is a 512-bit register, its content will
+  // be written to DRAM with SMC_WR commands.
+  program.add_inst(SMC_LI(wr_pattern, PATTERN_REG)); 
+  for(int i = 0 ; i < 16 ; i++)
+    program.add_inst(SMC_LDWD(PATTERN_REG,i));
+
+  program.add_inst(SMC_LI(0, BAR)); // Initialize BAR (bank address register) with 0
+
+  // add_label is used to add a label that can be used as a branch target to the program
+  program.add_label("BANK_BEGIN");
+
+  program.add_inst(SMC_LI(0, RAR)); // Initialize RAR with 0
+  
+  program.add_label("ROW_BEGIN");
+
+  program.add_inst(SMC_LI(0, CAR)); // Initialize CAR with 0
+
+  // Four DRAM commands are issued at the same time
+  // to DRAM by SoftMC.
+  
+  // PREcharge bank BAR and wait for tRP (11 NOPs, 11 SoftMC cycles) 
+  program.add_inst(
+    SMC_PRE(BAR, 0, 0, PC_UNDER_TEST),
+    SMC_NOP(), SMC_NOP(), SMC_NOP()
+  );
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+  
+  // ACT & wait for tRCD
+  program.add_inst(
+    SMC_ACT(BAR, 0, RAR, 0, PC_UNDER_TEST),
+    SMC_NOP(), SMC_NOP(), SMC_NOP()
+  );
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+
+  // We are now going to loop over all 512-bits in the row
+  // Since each row is 2^13 bytes we need to WR 128 times
+  program.add_inst(SMC_LI(NUM_COLS,NUM_COLS_REG));  // Load COL_SIZE register
+  program.add_inst(SMC_LI(0,LOOP_COLS));   // Load loop variable
+
+  /*
+    * Try to write a different pattern to each cache line
+    * in DRAM.
+    */
+  program.add_label("WR_BEGIN");
+
+  // Write to a whole row, increment CAR per WR
+  program.add_inst(
+    SMC_WRITE(BAR, 0, CAR, 1, PC_UNDER_TEST, 0),
+    SMC_NOP(), SMC_NOP(), SMC_NOP()
+  );    
+  
+  // Reload wide reg with the new data pattern
+  program.add_inst(SMC_SRC(PATTERN_REG,PATTERN_REG));
+  for(int i = 0 ; i < 16 ; i++)
+    program.add_inst(SMC_LDWD(PATTERN_REG,i));
+  
+  program.add_inst(SMC_ADDI(LOOP_COLS,1,LOOP_COLS));
+  // add a branch lower instruction which will jump to WR_BEGIN
+  // as long as LOOP_COLS<NUM_COLS, to iterate over the whole row
+  program.add_branch(program.BR_TYPE::BL,LOOP_COLS,NUM_COLS_REG, "WR_BEGIN");
+  // END "WR_BEGIN" LOOP
+
+  // Add more randomness to the PATTERN_REG by multiplying it by 3
+  // This happens when a row is written to
+  program.add_inst(SMC_MV(PATTERN_REG,TEMP_PATTERN_REG));
+  program.add_inst(SMC_ADD(PATTERN_REG,TEMP_PATTERN_REG,PATTERN_REG));
+  program.add_inst(SMC_ADD(PATTERN_REG,TEMP_PATTERN_REG,PATTERN_REG));
+  for(int i = 0 ; i < 16 ; i++)
+    program.add_inst(SMC_LDWD(PATTERN_REG,i));
+  
+  // Wait for t(write-precharge)
+  // & precharge the open bank
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+  program.add_inst(
+    SMC_PRE(BAR, 0, 0, PC_UNDER_TEST),
+    SMC_NOP(), SMC_NOP(), SMC_NOP()
+  );
+
+  program.add_inst(SMC_LI(0, CAR)); // reload CAR we are now going to read the whole row
+  
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+
+  // ACT & wait for tRCD
+  program.add_inst(
+    SMC_ACT(BAR, 0, RAR, 0, PC_UNDER_TEST),
+    SMC_NOP(), SMC_NOP(), SMC_NOP()
+  );
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+
+  // Read from a whole row
+  // Essentially this can be imagined as an unrolled read whole row loop
+  for(int i = 0 ; i < 32 ; i++){
+    program.add_inst(
+      SMC_READ(BAR, 0, CAR, 1, PC_UNDER_TEST, 0),
+      SMC_NOP(), SMC_NOP(), SMC_NOP()
+    );
+    program.add_inst(all_nops());
+  }
+  
+  // Wait for t(read-precharge)
+  // & precharge the open bank
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+  program.add_inst(
+    SMC_PRE(BAR, 0, 0, PC_UNDER_TEST),
+    SMC_NOP(), SMC_NOP(), SMC_NOP()
+  );
+
+  // Increment row address by one since we are finished with this row
+  program.add_inst(SMC_ADDI(RAR,1,RAR));
+  program.add_branch(program.BR_TYPE::BL,RAR,NUM_ROWS_REG, "ROW_BEGIN");
+  program.add_inst(SMC_ADDI(BAR,1,BAR));
+  program.add_branch(program.BR_TYPE::BL,BAR,NUM_BANKS_REG,"BANK_BEGIN");
+  
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+  program.add_inst(all_nops());
+
+  program.add_inst(SMC_ADDI(LOOP_CH, 1, LOOP_CH));
+  program.add_branch(program.BR_TYPE::BL, LOOP_CH, NUM_CH_REG,"CH_BEGIN");
+
+  // program.dump_registers();
+
+  // Tells SoftMC that the user program has ended
+  program.add_inst(SMC_END());
+
+  // Transfer the program to the FPGA board
+  platform.execute(program);
+  printf("Program Ready\n");
+  program.pretty_print();
+  printf("Sent instructions\n");
+  printf("Testing PC: %d\n", PC_UNDER_TEST);
+
+  // Read count
+  int rc = 0;
+
+  uint64_t err_count = 0;  
+  uint32_t pattern = wr_pattern;  
+
+  for (int ch = 0 ; ch < NUM_CH ; ch++)
+  {  
+    pattern = wr_pattern;  
+    // Below loop reads continuously from the FPGA board
+    // since we know in what order we are issuing our reads
+    // we know what physical address they are reading
+    while(1)
+    {
+      // Read the data from the FPGA board
+      platform.receiveData(buf, ROW_SIZE * 2);
+
+      for(int j = 0 ; j < 32 ; j++){
+        // printf("Whole pattern for cacheline %d: %x\n", j, pattern);
+        for (int k = 0 ; k < 32 ; k++){ // each byte in cache line
+          uint8_t mini_patt = pattern >> ((uint32_t)(k%4)*8);
+          if(mini_patt != (uint8_t)buf[j*64 + (PC_UNDER_TEST*32) + k]){ // Skip every other 32 bytes because they come from a different PC
+            err_count++;
+            fprintf(stderr,"t1: Pattern mismatch! CH:%d Bank: %d Row: %d CacheLine: %d Byte: %d Expect: %x Read: %x\n", 
+                  ch, rc/(1024*32), rc%(1024*32), j, k, mini_patt, (uint8_t)buf[j*64 + k]);
+            //fprintf(stderr,"%d %d %d %x\n", 
+              //    rc/(1024*32), rc%(1024*32), j*64 + k, mini_patt^(uint8_t)buf[j*64 + k]);
+          }
+        }
+        //shift pattern at each row iteration
+        uint32_t imd  = pattern >> (uint32_t)1;
+        uint32_t imd2 = pattern << (uint32_t)31;
+        pattern = imd | imd2; 
+      }
+      pattern = pattern*(uint32_t)3;
+      rc++;
+      if(rc % (NUM_ROWS) == 0)
+        printf("Bank %d finished!\n", (rc/(NUM_ROWS)));
+      if(rc % (NUM_ROWS * NUM_BANKS) == 0)
+        break;
+    }
+    printf("Channel %d finished!\n", ch + 1);
+  }
+  printf("%ld out of %ld bytes have errors, last pattern: 0x%x\n",
+    err_count, (uint64_t)NUM_CH*NUM_BANKS*NUM_ROWS*1024, pattern);
+  //platform.readRegisterDump();
+}
diff --git a/sources/hdl/header_verilog/encoding.vh b/sources/hdl/header_verilog/encoding.vh
index f5d5697..c0265d6 100644
--- a/sources/hdl/header_verilog/encoding.vh
+++ b/sources/hdl/header_verilog/encoding.vh
@@ -20,13 +20,15 @@
 `define DDR_CODE_OFFSET   12
 `define DEC_CAR           4
 `define DEC_BAR           0
+`define DEC_SEL_CH        0
 `define DEC_RAR           4
 `define DEC_INC_BAR       10
 `define DEC_PRE_ALL       11
 `define DEC_INC_RAR       11
 `define DEC_INC_CAR       11
 `define DEC_AP            9
-`define DEC_BL4           8
+`define DEC_BL4           8 // deprecated
+`define DEC_RANK          8
 // function codes - exe
 `define ADD               0
 `define ADDI              1
@@ -54,6 +56,7 @@
 `define PRE               2
 `define ACT               3
 `define ZQ                4
+`define SEL_CH            4
 `define REF               5
 `define NOP               7
 
@@ -63,6 +66,7 @@
 `define IS_PRE            2
 `define IS_ACT            3
 `define IS_ZQ             4
+`define IS_SEL_CH         4
 `define IS_REF            5
 `define CAR               6  // column address register identifier
 `define RAR               10 // row address register identifier
@@ -72,10 +76,12 @@
 `define INC_RAR           20
 `define INC_BAR           21
 `define IS_NOP            22
-`define IS_BL4            23
+`define IS_BL4            23 // deprecated
+`define IS_RANK           23 
 `define DO_AP             24
 `define IS_SRE            25
 `define IS_SRX            26
+`define HBM_CHANNEL       27  // HBM channel identifier
 // SoftMC exe uops
 `define IS_ADD            0
 `define IS_SUB            1
diff --git a/sources/hdl/header_verilog/parameters.vh b/sources/hdl/header_verilog/parameters.vh
index 916c973..f60c2ad 100644
--- a/sources/hdl/header_verilog/parameters.vh
+++ b/sources/hdl/header_verilog/parameters.vh
@@ -3,12 +3,13 @@
 `define IMEM_ADDR_WIDTH 11
 
 // Decode - Execute
-`define DDR_UOP_WIDTH 27
+`define DDR_UOP_WIDTH 31
 `define EXE_UOP_WIDTH 62
 `define BG_WIDTH       2
 `define BANK_WIDTH     2
 `define COL_WIDTH     10
 `define ROW_WIDTH     17
+`define HBM_CH_WIDTH   4
 
 //Frontend
 `define XDMA_AXI_DATA_WIDTH 256
diff --git a/sources/hdl/verilog/ddr_pipeline.v b/sources/hdl/verilog/ddr_pipeline.v
index f8ae5aa..3fea5bc 100644
--- a/sources/hdl/verilog/ddr_pipeline.v
+++ b/sources/hdl/verilog/ddr_pipeline.v
@@ -12,23 +12,25 @@ module ddr_pipeline(
   input [`DDR_UOP_WIDTH*4-1:0]  ddr_uop,
 
   // ddr_pipeline <-> outer DDRX IP interface
-  output [3:0]                ddr_write,
-  output [3:0]                ddr_read,
-  output [3:0]                ddr_pre,
-  output [3:0]                ddr_act,
-  output [3:0]                ddr_ref,
-  output [3:0]                ddr_zq,
-  output [3:0]                ddr_nop,
-  output [3:0]                ddr_sre,
-  output [3:0]                ddr_srx,
-  output [3:0]                ddr_ap,
-  output [3:0]                ddr_pall,
-  output [3:0]                ddr_half_bl,
-  output [4*`BG_WIDTH-1:0]    ddr_bg, 
-  output [4*`BANK_WIDTH-1:0]  ddr_bank,
-  output [4*`COL_WIDTH-1:0]   ddr_col,
-  output [4*`ROW_WIDTH-1:0]   ddr_row,
-  output [511:0]              ddr_wdata,
+  output [3:0]                  ddr_write,
+  output [3:0]                  ddr_read,
+  output [3:0]                  ddr_pre,
+  output [3:0]                  ddr_act,
+  output [3:0]                  ddr_ref,
+  output [3:0]                  ddr_zq,
+  output [3:0]                  ddr_nop,
+  output [3:0]                  ddr_sre,
+  output [3:0]                  ddr_srx,
+  output [3:0]                  ddr_ap,
+  output [3:0]                  ddr_pall,
+  output [3:0]                  ddr_half_bl,
+  output [3:0]                  ddr_rank,
+  output [4*`HBM_CH_WIDTH-1:0]  hbm_ch, 
+  output [4*`BG_WIDTH-1:0]      ddr_bg, 
+  output [4*`BANK_WIDTH-1:0]    ddr_bank,
+  output [4*`COL_WIDTH-1:0]     ddr_col,
+  output [4*`ROW_WIDTH-1:0]     ddr_row,
+  output [511:0]                ddr_wdata,
 
   // ddr_pipeline <-> regfile interface
   input  [511:0]                        wide_reg,
@@ -59,23 +61,25 @@ module ddr_pipeline(
   reg [7:0] s2_update_en; // which registers will we update
   reg [8*32-1:0] s2_update_val; 
 
-  reg [3:0]                ddr_write_ns, ddr_write_r;
-  reg [3:0]                ddr_read_ns, ddr_read_r; 
-  reg [3:0]                ddr_pre_ns, ddr_pre_r;
-  reg [3:0]                ddr_act_ns, ddr_act_r;
-  reg [3:0]                ddr_ref_ns, ddr_ref_r;
-  reg [3:0]                ddr_sre_ns, ddr_sre_r;
-  reg [3:0]                ddr_srx_ns, ddr_srx_r;
-  reg [3:0]                ddr_zq_ns, ddr_zq_r;
-  reg [3:0]                ddr_nop_ns, ddr_nop_r;
-  reg [3:0]                ddr_ap_ns, ddr_ap_r;
-  reg [3:0]                ddr_pall_ns, ddr_pall_r;
-  reg [3:0]                ddr_half_bl_ns, ddr_half_bl_r;
-  reg [4*`BG_WIDTH-1:0]    ddr_bg_ns, ddr_bg_r;
-  reg [4*`BANK_WIDTH-1:0]  ddr_bank_ns, ddr_bank_r;
-  reg [4*`COL_WIDTH-1:0]   ddr_col_ns, ddr_col_r;
-  reg [4*`ROW_WIDTH-1:0]   ddr_row_ns, ddr_row_r;
-  reg [511:0]              ddr_data_r;
+  reg [3:0]                   ddr_write_ns, ddr_write_r;
+  reg [3:0]                   ddr_read_ns, ddr_read_r; 
+  reg [3:0]                   ddr_pre_ns, ddr_pre_r;
+  reg [3:0]                   ddr_act_ns, ddr_act_r;
+  reg [3:0]                   ddr_ref_ns, ddr_ref_r;
+  reg [3:0]                   ddr_sre_ns, ddr_sre_r;
+  reg [3:0]                   ddr_srx_ns, ddr_srx_r;
+  reg [3:0]                   ddr_zq_ns, ddr_zq_r;
+  reg [3:0]                   ddr_nop_ns, ddr_nop_r;
+  reg [3:0]                   ddr_ap_ns, ddr_ap_r;
+  reg [3:0]                   ddr_pall_ns, ddr_pall_r;
+  reg [3:0]                   ddr_half_bl_ns, ddr_half_bl_r;
+  reg [4*`HBM_CH_WIDTH-1:0]   hbm_ch_ns, hbm_ch_r;
+  reg [3:0]                   ddr_rank_ns, ddr_rank_r;
+  reg [4*`BG_WIDTH-1:0]       ddr_bg_ns, ddr_bg_r;
+  reg [4*`BANK_WIDTH-1:0]     ddr_bank_ns, ddr_bank_r;
+  reg [4*`COL_WIDTH-1:0]      ddr_col_ns, ddr_col_r;
+  reg [4*`ROW_WIDTH-1:0]      ddr_row_ns, ddr_row_r;
+  reg [511:0]                 ddr_data_r;
 
   assign update_vals  = s2_update_val;
   assign update_en    = s2_update_en;
@@ -95,6 +99,8 @@ module ddr_pipeline(
   assign ddr_ap       = ddr_ap_r;
   assign ddr_pall     = ddr_pall_r;
   assign ddr_half_bl  = ddr_half_bl_r;
+  assign ddr_rank     = ddr_rank_r;
+  assign hbm_ch       = hbm_ch_r;
   assign ddr_bg       = ddr_bg_r;
   assign ddr_bank     = ddr_bank_r;
   assign ddr_col      = ddr_col_r;
@@ -131,8 +137,11 @@ module ddr_pipeline(
       ddr_sre_ns[i]     = s2_uop[i][`IS_SRE];
       ddr_srx_ns[i]     = s2_uop[i][`IS_SRX];
       ddr_ap_ns[i]      = s2_uop[i][`DO_AP] & (s2_uop[i][`IS_WRITE] | s2_uop[i][`IS_READ]);
+      ddr_rank_ns[i]    = s2_uop[i][`IS_RANK];
       ddr_half_bl_ns[i] = s2_uop[i][`IS_BL4] & (s2_uop[i][`IS_WRITE] | s2_uop[i][`IS_READ]);
       ddr_nop_ns[i]     = s2_valid ? s2_uop[i][`IS_NOP] : {4{`HIGH}};
+      hbm_ch_ns[i*`HBM_CH_WIDTH +: `HBM_CH_WIDTH] 
+        = s2_uop[i][`HBM_CHANNEL +: `HBM_CH_WIDTH];
       // stage 2 address calculation
       ddr_row_ns[i*`ROW_WIDTH +: `ROW_WIDTH] 
         = reg_vals[i*64+32 +: `ROW_WIDTH];
@@ -177,6 +186,8 @@ module ddr_pipeline(
       ddr_nop_r      <= {4{`HIGH}};
       ddr_ap_r       <= {4{`LOW}};
       ddr_half_bl_r  <= {4{`LOW}};
+      ddr_rank_r     <= {4{`LOW}};
+      hbm_ch_r       <= {4*`HBM_CH_WIDTH{`LOW}};
     end
     else begin
       reg_ids_r <= reg_ids_ns;
@@ -194,7 +205,9 @@ module ddr_pipeline(
       ddr_nop_r      <= ddr_nop_ns;
       ddr_ap_r       <= ddr_ap_ns;
       ddr_pall_r     <= ddr_pall_ns;
-      ddr_half_bl_r  <= ddr_half_bl_ns;
+      ddr_half_bl_r  <= {4{`LOW}}; // deprecated
+      ddr_rank_r     <= ddr_rank_ns;
+      hbm_ch_r       <= hbm_ch_ns;
       ddr_bg_r       <= ddr_bg_ns;
       ddr_bank_r     <= ddr_bank_ns;
       ddr_col_r      <= ddr_col_ns;
diff --git a/sources/hdl/verilog/decode_stage.v b/sources/hdl/verilog/decode_stage.v
index 7f40cba..754a2d9 100644
--- a/sources/hdl/verilog/decode_stage.v
+++ b/sources/hdl/verilog/decode_stage.v
@@ -77,6 +77,7 @@ module decode_stage(
               ddr_uop_ns[i][`BAR+:4]   = ddr_insts[i][`DEC_BAR+:4];
               ddr_uop_ns[i][`DO_AP]    = ddr_insts[i][`DEC_AP];
               ddr_uop_ns[i][`IS_BL4]   = ddr_insts[i][`DEC_BL4];
+              ddr_uop_ns[i][`IS_RANK]  = ddr_insts[i][`DEC_RANK];
             end
             `READ: begin
               ddr_uop_ns[i][`IS_READ]  = `HIGH;
@@ -86,12 +87,14 @@ module decode_stage(
               ddr_uop_ns[i][`BAR+:4]   = ddr_insts[i][`DEC_BAR+:4];
               ddr_uop_ns[i][`DO_AP]    = ddr_insts[i][`DEC_AP];
               ddr_uop_ns[i][`IS_BL4]   = ddr_insts[i][`DEC_BL4];
+              ddr_uop_ns[i][`IS_RANK]  = ddr_insts[i][`DEC_RANK];              
             end
             `PRE: begin
               ddr_uop_ns[i][`IS_PRE]   = `HIGH;
               ddr_uop_ns[i][`PRE_ALL]  = ddr_insts[i][`DEC_PRE_ALL];
               ddr_uop_ns[i][`INC_BAR]  = ddr_insts[i][`DEC_INC_BAR];
               ddr_uop_ns[i][`BAR+:4]   = ddr_insts[i][`DEC_BAR+:4];
+              ddr_uop_ns[i][`IS_RANK]  = ddr_insts[i][`DEC_RANK];              
             end
             `ACT: begin
               ddr_uop_ns[i][`IS_ACT]   = `HIGH;
@@ -99,15 +102,30 @@ module decode_stage(
               ddr_uop_ns[i][`INC_BAR]  = ddr_insts[i][`DEC_INC_BAR];
               ddr_uop_ns[i][`RAR+:4]   = ddr_insts[i][`DEC_RAR+:4];
               ddr_uop_ns[i][`BAR+:4]   = ddr_insts[i][`DEC_BAR+:4];
+              ddr_uop_ns[i][`IS_RANK]  = ddr_insts[i][`DEC_RANK];
             end
+`ifndef HBM_BENDER
             `ZQ: begin
               ddr_uop_ns[i][`IS_ZQ]    = `HIGH;
+              ddr_uop_ns[i][`IS_RANK]  = ddr_insts[i][`DEC_RANK];              
             end
+`else
+            `SEL_CH: begin
+              // this is redundant and needs fixing
+              // we use ZQ to select channels in the HBM version
+              ddr_uop_ns[i][`IS_ZQ]          = `HIGH;
+              ddr_uop_ns[i][`IS_RANK]  = ddr_insts[i][`DEC_RANK];              
+              ddr_uop_ns[i][`HBM_CHANNEL +: `HBM_CH_WIDTH] = ddr_insts[i][`DEC_SEL_CH +: `HBM_CH_WIDTH];
+              ddr_uop_ns[i][`IS_SEL_CH] = `HIGH;
+            end
+`endif
             `REF: begin
               ddr_uop_ns[i][`IS_REF]   = `HIGH;
+              ddr_uop_ns[i][`IS_RANK]  = ddr_insts[i][`DEC_RANK];              
             end
             `NOP: begin
               ddr_uop_ns[i][`IS_NOP]   = `HIGH;
+              ddr_uop_ns[i][`IS_RANK]  = ddr_insts[i][`DEC_RANK];            
             end
           endcase
         end
diff --git a/sources/hdl/verilog/dll_toggler.v b/sources/hdl/verilog/dll_toggler.v
index a6cb5bf..9d51679 100644
--- a/sources/hdl/verilog/dll_toggler.v
+++ b/sources/hdl/verilog/dll_toggler.v
@@ -1,5 +1,5 @@
 `include "parameters.vh"
-
+`define CKE_WIDTH 2
 module dll_toggler #(parameter CKE_WIDTH = 1, RANK_WIDTH = 1, DQ_WIDTH = 64, DRAM_CMD_SLOTS = 4,
 										DATA_BUF_ADDR_WIDTH = 5, DBUF_WIDTH = 4, DQ_BURST = 8)
 
@@ -12,7 +12,7 @@ module dll_toggler #(parameter CKE_WIDTH = 1, RANK_WIDTH = 1, DQ_WIDTH = 64, DRA
   
   // ----------  DDR4 Signals  ----------
   output [7:0]                                      mc_ACT_n,  // DRAM ACT_n command signal for four DRAM clock cycles.
-  output [`ADDR_WIDTH*8-1:0]                        mc_ADR,    // DRAM address. There are 8 bits in the fabric interface for each address bit on the DRAM bus.
+  output [17*8-1:0]                                 mc_ADR,    // DRAM address. There are 8 bits in the fabric interface for each address bit on the DRAM bus.
   output [`BANK_WIDTH*8-1:0]                        mc_BA,     // DRAM bank address. 8 bits for each DRAM bank address.
   output [`BG_WIDTH*8-1:0]                          mc_BG,     // DRAM bank group address.
   output [`CS_WIDTH*8-1:0]                          mc_CS_n,   // DRAM CS_n
@@ -25,7 +25,7 @@ module dll_toggler #(parameter CKE_WIDTH = 1, RANK_WIDTH = 1, DQ_WIDTH = 64, DRA
     
   wire [13:0] MR1_CONF = 14'b00001100000000;  
     
-  reg [`ADDR_WIDTH*8-1:0]       ADR_ns, ADR_r;
+  reg [17*8-1:0]       ADR_ns, ADR_r;
   reg [`BANK_WIDTH*8-1:0]       BA_ns, BA_r;
   reg [`BG_WIDTH*8-1:0]         BG_ns, BG_r;
   reg [`CS_WIDTH*8-1:0]         CS_n_ns, CS_n_r;
@@ -83,7 +83,7 @@ module dll_toggler #(parameter CKE_WIDTH = 1, RANK_WIDTH = 1, DQ_WIDTH = 64, DRA
   always @* begin
     // Set bank and bank group signals
     dllt_done  = `LOW;
-    ADR_ns     = {`ADDR_WIDTH*8{`HIGH}};
+    ADR_ns     = {17*8{`HIGH}};
     BG_ns      = {`BG_WIDTH*8{`LOW}};
     BA_ns      = {`BANK_WIDTH*8{`LOW}};
     CS_n_ns    = {`CS_WIDTH*8{`HIGH}}; // by default we don't issue any commands
@@ -95,9 +95,9 @@ module dll_toggler #(parameter CKE_WIDTH = 1, RANK_WIDTH = 1, DQ_WIDTH = 64, DRA
       IDLE_S: begin
         if(toggle_valid) begin
           CS_n_ns[1:0] = {2*`CS_WIDTH{`LOW}};
-          ADR_ns[`ADDR_WIDTH*8-3*8 +: 2] = {2{`LOW}};  // WE
-          ADR_ns[`ADDR_WIDTH*8-2*8 +: 2] = {2{`HIGH}}; // ~CAS
-          ADR_ns[`ADDR_WIDTH*8-8   +: 2] = {2{`LOW}};  // RAS
+          ADR_ns[17*8-3*8 +: 2] = {2{`LOW}};  // WE
+          ADR_ns[17*8-2*8 +: 2] = {2{`HIGH}}; // ~CAS
+          ADR_ns[17*8-8   +: 2] = {2{`LOW}};  // RAS
           ADR_ns[10*8 +: 2]              = {2{`HIGH}}; // Pre ALL
           wait_ns                        = T_PRECHARGE;
           state_ns                       = IDLE_WAIT_S;
@@ -111,9 +111,9 @@ module dll_toggler #(parameter CKE_WIDTH = 1, RANK_WIDTH = 1, DQ_WIDTH = 64, DRA
       end
       SET_MR_1_S: begin
         CS_n_ns[1:0] = {2*`CS_WIDTH{`LOW}};
-        ADR_ns[`ADDR_WIDTH*8-3*8 +: 2] = {2{`LOW}};  // WE
-        ADR_ns[`ADDR_WIDTH*8-2*8 +: 2] = {2{`LOW}};  // CAS
-        ADR_ns[`ADDR_WIDTH*8-8   +: 2] = {2{`LOW}};  // RAS
+        ADR_ns[17*8-3*8 +: 2] = {2{`LOW}};  // WE
+        ADR_ns[17*8-2*8 +: 2] = {2{`LOW}};  // CAS
+        ADR_ns[17*8-8   +: 2] = {2{`LOW}};  // RAS
         for(adr_bit_i = 0 ; adr_bit_i < 14 ; adr_bit_i = adr_bit_i + 1) begin
             ADR_ns[adr_bit_i*8 +: 2] =
                       {2{MR1_CONF[adr_bit_i]}};
@@ -133,9 +133,9 @@ module dll_toggler #(parameter CKE_WIDTH = 1, RANK_WIDTH = 1, DQ_WIDTH = 64, DRA
       ENTER_SELF_REF_S: begin
         CKE_ns = `LOW;
         CS_n_ns[1:0] = {2*`CS_WIDTH{`LOW}};
-        ADR_ns[`ADDR_WIDTH*8-3*8 +: 2] = {2{`HIGH}};  // ~WE
-        ADR_ns[`ADDR_WIDTH*8-2*8 +: 2] = {2{`LOW}};  // CAS
-        ADR_ns[`ADDR_WIDTH*8-8   +: 2] = {2{`LOW}};  // RAS
+        ADR_ns[17*8-3*8 +: 2] = {2{`HIGH}};  // ~WE
+        ADR_ns[17*8-2*8 +: 2] = {2{`LOW}};  // CAS
+        ADR_ns[17*8-8   +: 2] = {2{`LOW}};  // RAS
         state_ns = WAIT_ENTER_SELF_REF_S;
         wait_ns  = T_CKSRE;
       end
@@ -161,9 +161,9 @@ module dll_toggler #(parameter CKE_WIDTH = 1, RANK_WIDTH = 1, DQ_WIDTH = 64, DRA
       EXIT_SELF_REF_S: begin
         CKE_ns = {`CKE_WIDTH*8{`HIGH}};
         CS_n_ns[7:0] = {8*`CS_WIDTH{`HIGH}};
-        ADR_ns[`ADDR_WIDTH*8-3*8 +: 2] = {2{`HIGH}};  // ~WE
-        ADR_ns[`ADDR_WIDTH*8-2*8 +: 2] = {2{`HIGH}};  // CAS
-        ADR_ns[`ADDR_WIDTH*8-8   +: 2] = {2{`HIGH}};  // RAS
+        ADR_ns[17*8-3*8 +: 2] = {2{`HIGH}};  // ~WE
+        ADR_ns[17*8-2*8 +: 2] = {2{`HIGH}};  // CAS
+        ADR_ns[17*8-8   +: 2] = {2{`HIGH}};  // RAS
         state_ns = WAIT_EXIT_SELF_REF_S;
         wait_ns  = T_XS;
       end
@@ -186,7 +186,7 @@ module dll_toggler #(parameter CKE_WIDTH = 1, RANK_WIDTH = 1, DQ_WIDTH = 64, DRA
       wait_r    <= `LOW;
       clk_sel_r <= `LOW;
       CKE_r     <= {`CKE_WIDTH*8{`HIGH}};
-      ADR_r      = {`ADDR_WIDTH*8{`LOW}};
+      ADR_r      = {17*8{`LOW}};
       BG_r       = {`BG_WIDTH*8{`LOW}};
       BA_r       = {`BANK_WIDTH*8{`LOW}};
       CS_n_r     = {`CS_WIDTH*8{`HIGH}};
diff --git a/sources/hdl/verilog/execute_stage.v b/sources/hdl/verilog/execute_stage.v
index 0b690f8..5ca8d38 100644
--- a/sources/hdl/verilog/execute_stage.v
+++ b/sources/hdl/verilog/execute_stage.v
@@ -18,23 +18,25 @@ module execute_stage(
   output [`IMEM_ADDR_WIDTH-1:0] br_target,
 
   // execute <-> outer DDRX IP interface
-  output [3:0]                ddr_write,
-  output [3:0]                ddr_read,
-  output [3:0]                ddr_pre,
-  output [3:0]                ddr_act,
-  output [3:0]                ddr_ref,
-  output [3:0]                ddr_sre,
-  output [3:0]                ddr_srx,
-  output [3:0]                ddr_zq,
-  output [3:0]                ddr_nop,
-  output [3:0]                ddr_ap,
-  output [3:0]                ddr_pall,
-  output [3:0]                ddr_half_bl,
-  output [4*`BG_WIDTH-1:0]    ddr_bg, 
-  output [4*`BANK_WIDTH-1:0]  ddr_bank,
-  output [4*`COL_WIDTH-1:0]   ddr_col,
-  output [4*`ROW_WIDTH-1:0]   ddr_row,
-  output [511:0]              ddr_wdata,
+  output [3:0]                  ddr_write,
+  output [3:0]                  ddr_read,
+  output [3:0]                  ddr_pre,
+  output [3:0]                  ddr_act,
+  output [3:0]                  ddr_ref,
+  output [3:0]                  ddr_sre,
+  output [3:0]                  ddr_srx,
+  output [3:0]                  ddr_zq,
+  output [3:0]                  ddr_nop,
+  output [3:0]                  ddr_ap,
+  output [3:0]                  ddr_pall,
+  output [3:0]                  ddr_half_bl,
+  output [3:0]                  ddr_rank,
+  output [4*`HBM_CH_WIDTH-1:0]  hbm_ch,
+  output [4*`BG_WIDTH-1:0]      ddr_bg, 
+  output [4*`BANK_WIDTH-1:0]    ddr_bank,
+  output [4*`COL_WIDTH-1:0]     ddr_col,
+  output [4*`ROW_WIDTH-1:0]     ddr_row,
+  output [511:0]                ddr_wdata,
  
   // execute <-> reg file if
   output                              rf_wide_wen,
@@ -86,6 +88,8 @@ module execute_stage(
     .ddr_ap(ddr_ap),
     .ddr_pall(ddr_pall),
     .ddr_half_bl(ddr_half_bl),
+    .hbm_ch(hbm_ch),
+    .ddr_rank(ddr_rank),
     .ddr_bg(ddr_bg), 
     .ddr_bank(ddr_bank),
     .ddr_col(ddr_col),
diff --git a/sources/hdl/verilog/frontend.v b/sources/hdl/verilog/frontend.v
index 9225bd9..c1ce196 100644
--- a/sources/hdl/verilog/frontend.v
+++ b/sources/hdl/verilog/frontend.v
@@ -25,6 +25,10 @@ module frontend#(parameter SIM_MEM = "false")(
   output                        valid_out,
   output [`IMEM_ADDR_WIDTH-1:0] addr_out,
   output                        ready_in,  
+  
+  `ifdef HBM_BENDER
+  output                              hbm_temp_rd,
+  `endif
 
   // frontend <-> xdma interface
   input  [`XDMA_AXI_DATA_WIDTH-1:0]   h2c_tdata_0,
@@ -37,6 +41,7 @@ module frontend#(parameter SIM_MEM = "false")(
   output                              per_rd_init,
   output                              per_zq_init,
   output                              per_ref_init
+  
   );
 
   reg[31:0] delay_fin;
@@ -125,6 +130,10 @@ module frontend#(parameter SIM_MEM = "false")(
   reg [`IMEM_RD_LATENCY-1:0] valid_out_sr;
   reg [(`IMEM_RD_LATENCY * `IMEM_ADDR_WIDTH)-1:0] addr_out_sr;
 
+  reg hbm_temp_rd_ns, hbm_temp_rd_r;
+  
+  assign hbm_temp_rd = hbm_temp_rd_r;
+
   assign user_rst     = (|rst_ctr_r);
 
   // imem <-> xdma interface
@@ -150,6 +159,9 @@ module frontend#(parameter SIM_MEM = "false")(
   always @* begin
     aref_en_valid   = `LOW;
     aref_en         = `LOW;
+    `ifdef HBM_BENDER
+    hbm_temp_rd_ns  = `LOW;
+    `endif
     state_ns        = state_r;
     xfer_ctr_ns     = xfer_ctr_r;
     rst_ctr_ns      = {5{`LOW}};
@@ -182,6 +194,12 @@ module frontend#(parameter SIM_MEM = "false")(
             aref_en       = h2c_tdata_0[0];
             state_ns      = IDLE_S;
           end
+          `ifdef HBM_BENDER
+          else if(h2c_tdata_0[`INSTR_WIDTH+4]) begin // read HBM temp
+            hbm_temp_rd_ns  = `HIGH;
+            state_ns        = IDLE_S;
+          end
+          `endif
           else begin
             xfer_ctr_ns = xfer_ctr_r + 1;
             if(h2c_tlast_0) begin
@@ -216,6 +234,9 @@ module frontend#(parameter SIM_MEM = "false")(
         rst_ctr_r <= rst_ctr_r - 1;
       else
         rst_ctr_r <= 0;
+      `ifdef HBM_BENDER
+      hbm_temp_rd_r <= 0;
+      `endif
     end
     else begin
       state_r      <= state_ns;
@@ -230,6 +251,9 @@ module frontend#(parameter SIM_MEM = "false")(
         valid_out_sr[`IMEM_RD_LATENCY-1:0] <= valid_out_sr >> 1;
         addr_out_sr[(`IMEM_RD_LATENCY-1)*`IMEM_ADDR_WIDTH-1:0] 
                          <= addr_out_sr >> `IMEM_ADDR_WIDTH; 
+      `endif      
+      `ifdef HBM_BENDER
+      hbm_temp_rd_r <= hbm_temp_rd_ns;
       `endif
     end
   end
diff --git a/sources/hdl/verilog/maintenance_controller.v b/sources/hdl/verilog/maintenance_controller.v
index 9354d17..169833a 100644
--- a/sources/hdl/verilog/maintenance_controller.v
+++ b/sources/hdl/verilog/maintenance_controller.v
@@ -172,8 +172,12 @@ module maintenance_controller#(parameter tCK = 1500)(
 
   wire periodic_rd_timer_one = maint_prescaler_tick_r_lcl && (periodic_rd_timer_r == ONE[0+:PERIODIC_RD_TIMER_WIDTH]);
 
+  `ifndef HBM_BENDER
   wire periodic_rd_request = ~rst && (/*((PERIODIC_RD_TIMER_DIV != 0) && ~dfi_init_complete) ||*/
                       (~per_rd_ack && (per_rd_request || periodic_rd_timer_one)));
+  `else
+  wire periodic_rd_request = 1'b0;
+  `endif
 
   always @(posedge clk) begin
     if(~init_calib_complete)
@@ -221,8 +225,12 @@ module maintenance_controller#(parameter tCK = 1500)(
 
         always @(posedge clk) zq_request_r <= zq_request_ns;
 
-        always @(/*AS*/init_calib_complete or zq_request_r)
-          zq_request = init_calib_complete && zq_request_r;
+        always @(/*AS*/init_calib_complete or zq_request_r)       
+  `ifndef HBM_BENDER
+            zq_request = init_calib_complete && zq_request_r;
+  `else
+            zq_request = 1'b0;
+  `endif
       end // zq_request_logic
     end
   endgenerate
diff --git a/sources/hdl/verilog/readback_engine.v b/sources/hdl/verilog/readback_engine.v
index 496d83e..b5313af 100644
--- a/sources/hdl/verilog/readback_engine.v
+++ b/sources/hdl/verilog/readback_engine.v
@@ -25,6 +25,12 @@ module readback_engine(
   // engine <-> regfile if
   input [511:0] ddr_wdata, // to compare read data against
   
+  `ifdef HBM_BENDER
+  input hbm_temp_rd,
+  input [6:0] hbm0_temp,
+  input [6:0] hbm1_temp,
+  `endif
+
   // readback <-> XDMA if
   output [`XDMA_AXI_DATA_WIDTH-1:0]   c2h_tdata_0,  
   output                              c2h_tlast_0,
@@ -39,6 +45,9 @@ module readback_engine(
   localparam DIFF_MODE = 1;
   reg mode_r, mode_ns; // Switch between diff count and read modes
   
+  // used to ignore the reads from both ranks after a periodic read
+  reg rd_flag_r, rd_flag_ns;
+  
   reg rd_valid_r;
   reg ignore_read_r, ignore_read_ns;
   reg ignore_flush_r, ignore_flush_ns;
@@ -134,13 +143,21 @@ module readback_engine(
   wire rbf_empty, rbf_rd_valid, fifo_almost_full, fifo_valid;
   (*KEEP = "TRUE"*) wire rbf_full;
   (*KEEP = "TRUE"*) reg [19:0] dbg_rd_ctr;
+   
+  `ifdef HBM_BENDER
+  wire [15:0] hbm_temp_all = {1'b0, hbm1_temp, 1'b0, hbm0_temp};
+  wire [511:0] rdback_din = mode_r == READ_MODE ? hbm_temp_rd ? hbm_temp_all : {rd_data[0+:128], rd_data[128+:128], rd_data[256+:128], rd_data[384+:128]} : {dsr_out[255:0],dsr_out[511:256]};
+  `else
+  wire [511:0] rdback_din = mode_r == READ_MODE ? {rd_data[255:0],rd_data[511:256]} : {dsr_out[255:0],dsr_out[511:256]};
+  `endif
+  
   rdback_fifo rbf(
     .full(rbf_full),
     .prog_full(fifo_almost_full),
     .empty(rbf_empty),
-    .wr_en(mode_r == READ_MODE ? rd_valid && ~ignore_read_r: dsr_valid),
+    .wr_en(mode_r == READ_MODE ? (rd_valid | hbm_temp_rd) && ~ignore_read_r: dsr_valid),
     // shuffle data because fifo outputs them on wrong order
-    .din(mode_r == READ_MODE ? {rd_data[255:0],rd_data[511:256]} : {dsr_out[255:0],dsr_out[511:256]}),
+    .din(rdback_din),
     .rd_en(c2h_tready_0),
     .dout(c2h_tdata_0),
     .valid(fifo_valid),
@@ -159,12 +176,22 @@ module readback_engine(
     ignore_read_ns = ignore_read_r;
     ignore_flush_ns = ignore_flush_r;
     buffer_space_ns = buffer_space_r;
+    rd_flag_ns = rd_flag_r;
     if(per_rd_init || per_zq_init || per_ref_init) begin
       ignore_read_ns = per_rd_init;
       ignore_flush_ns = `HIGH;
     end
+    `ifdef DUAL_RANK_SELECT
+    if(rd_valid_r) begin
+      rd_flag_ns = ~rd_flag_ns;
+      if (~rd_flag_ns) begin
+        ignore_read_ns = `LOW;
+      end
+    end
+    `else
     if(rd_valid_r)
       ignore_read_ns = `LOW;
+    `endif
     proc_flush_ns = proc_flush_r;
     if(flush) begin
       if(ignore_flush_r)
diff --git a/sources/hdl/verilog/softmc_pipeline.v b/sources/hdl/verilog/softmc_pipeline.v
index 07c106e..43849f6 100644
--- a/sources/hdl/verilog/softmc_pipeline.v
+++ b/sources/hdl/verilog/softmc_pipeline.v
@@ -22,23 +22,25 @@ module softmc_pipeline(
   input                          ready_out,
   
   // ddr_pipeline <-> outer DDR module
-  output [3:0]                ddr_write,
-  output [3:0]                ddr_read,
-  output [3:0]                ddr_pre,
-  output [3:0]                ddr_act,
-  output [3:0]                ddr_ref,
-  output [3:0]                ddr_sre,
-  output [3:0]                ddr_srx,
-  output [3:0]                ddr_zq,
-  output [3:0]                ddr_nop,
-  output [3:0]                ddr_ap,
-  output [3:0]                ddr_pall,
-  output [3:0]                ddr_half_bl,
-  output [4*`BG_WIDTH-1:0]    ddr_bg, 
-  output [4*`BANK_WIDTH-1:0]  ddr_bank,
-  output [4*`COL_WIDTH-1:0]   ddr_col,
-  output [4*`ROW_WIDTH-1:0]   ddr_row,
-  output [511:0]              ddr_wdata
+  output [3:0]                  ddr_write,
+  output [3:0]                  ddr_read,
+  output [3:0]                  ddr_pre,
+  output [3:0]                  ddr_act,
+  output [3:0]                  ddr_ref,
+  output [3:0]                  ddr_sre,
+  output [3:0]                  ddr_srx,
+  output [3:0]                  ddr_zq,
+  output [3:0]                  ddr_nop,
+  output [3:0]                  ddr_ap,
+  output [3:0]                  ddr_pall,
+  output [3:0]                  ddr_half_bl,
+  output [3:0]                  ddr_rank,
+  output [4*`HBM_CH_WIDTH-1:0]  hbm_ch,
+  output [4*`BG_WIDTH-1:0]      ddr_bg, 
+  output [4*`BANK_WIDTH-1:0]    ddr_bank,
+  output [4*`COL_WIDTH-1:0]     ddr_col,
+  output [4*`ROW_WIDTH-1:0]     ddr_row,
+  output [511:0]                ddr_wdata
   );
   
   wire                        br_resolve;
@@ -149,6 +151,8 @@ module softmc_pipeline(
     .ddr_ap(ddr_ap),
     .ddr_pall(ddr_pall),
     .ddr_half_bl(ddr_half_bl),
+    .hbm_ch(hbm_ch),
+    .ddr_rank(ddr_rank),
     .ddr_bg(ddr_bg), 
     .ddr_bank(ddr_bank),
     .ddr_col(ddr_col),