diff options
Diffstat (limited to 'sources/hdl/verilog/readback_engine.v')
| -rw-r--r-- | sources/hdl/verilog/readback_engine.v | 244 |
1 files changed, 244 insertions, 0 deletions
diff --git a/sources/hdl/verilog/readback_engine.v b/sources/hdl/verilog/readback_engine.v new file mode 100644 index 0000000..496d83e --- /dev/null +++ b/sources/hdl/verilog/readback_engine.v @@ -0,0 +1,244 @@ +`include "parameters.vh" + +// Process data coming from DRAM before sending it to the host. +module readback_engine( + + // common signals + input clk, + input rst, + + // other control signals + input flush, + input read_seq_incoming, // next few instructions will read from DRAM + input [11:0] incoming_reads, // how many reads next few instructions will issue + output[11:0] buffer_space, // remaining buffer size + input switch_mode, + + // DRAM <-> engine if + input [511:0] rd_data, + input rd_valid, + + input per_rd_init, + input per_zq_init, + input per_ref_init, + + // engine <-> regfile if + input [511:0] ddr_wdata, // to compare read data against + + // readback <-> XDMA if + output [`XDMA_AXI_DATA_WIDTH-1:0] c2h_tdata_0, + output c2h_tlast_0, + output c2h_tvalid_0, + input c2h_tready_0, + output [`XDMA_AXI_DATA_WIDTH/8-1:0] c2h_tkeep_0 + + ); + + + localparam READ_MODE = 0; + localparam DIFF_MODE = 1; + reg mode_r, mode_ns; // Switch between diff count and read modes + + reg rd_valid_r; + reg ignore_read_r, ignore_read_ns; + reg ignore_flush_r, ignore_flush_ns; + + // Popcount computation part + reg[511:0] read_diff; + reg diff_valid; + always @(posedge clk) begin + if(rst) begin + read_diff <= 512'bX; + diff_valid <= `LOW; + end + read_diff <= rd_valid ? rd_data ^ ddr_wdata : read_diff; + diff_valid <= rd_valid && ~ignore_read_r && mode_r == DIFF_MODE ? `HIGH : `LOW; + end + + genvar pcs; // popcount modules + + wire[2:0] pc_out [127:0]; + reg[3:0] pc_out_l2 [63:0]; + reg[4:0] pc_out_l3 [31:0]; + reg[5:0] pc_out_l4 [15:0]; + reg[6:0] pc_out_l5 [7:0]; + reg[7:0] pc_out_l6 [3:0]; + reg[8:0] pc_out_l7 [1:0]; + reg[15:0] pop_count_value; + reg pop_count_valid; + + generate + for(pcs = 0 ; pcs < 128 ; pcs = pcs + 1) begin: gen_pcs + pop_count4 pci + ( + .in(read_diff[pcs*4 +: 4]), + .out(pc_out[pcs]) + ); + end + endgenerate + + integer l1, l2, l3, l4, l5, l6; + always @* begin + for(l1 = 0 ; l1 < 64 ; l1 = l1+1) + pc_out_l2[l1] = pc_out[2*l1] + pc_out[2*l1+1]; + for(l2 = 0 ; l2 < 32 ; l2 = l2+1) + pc_out_l3[l2] = pc_out_l2[2*l2] + pc_out_l2[2*l2+1]; + for(l3 = 0 ; l3 < 16 ; l3 = l3+1) + pc_out_l4[l3] = pc_out_l3[2*l3] + pc_out_l3[2*l3+1]; + for(l4 = 0 ; l4 < 8 ; l4 = l4+1) + pc_out_l5[l4] = pc_out_l4[2*l4] + pc_out_l4[2*l4+1]; + for(l5 = 0 ; l5 < 4 ; l5 = l5+1) + pc_out_l6[l5] = pc_out_l5[2*l5] + pc_out_l5[2*l5+1]; + for(l6 = 0 ; l6 < 2 ; l6 = l6+1) + pc_out_l7[l6] = pc_out_l6[2*l6] + pc_out_l6[2*l6+1]; + end + + always @(posedge clk) begin + if(rst) begin + pop_count_value <= 16'bX; + pop_count_valid <= `LOW; + end + else begin + pop_count_value <= diff_valid ? pc_out_l7[0] + pc_out_l7[1] : pop_count_value; + pop_count_valid <= diff_valid ? `HIGH : `LOW; + end + end + + wire[511:0] dsr_out; + wire dsr_valid; + // We put popcounted data into a shift register + // to fill up 512 bit I/O fifo. + diff_shift_reg dsr( + .clk(clk), + .rst(rst), + + .in(pop_count_value), + .in_valid(pop_count_valid), + + .flush(flush&~ignore_flush_r), + + .out(dsr_out), + .out_valid(dsr_valid) + ); + // End popcount computation part + + // Count up to 1024 32-byte transfers + reg[9:0] xctr_r; + + reg tlast; // indicating c2h's last transfer + + // We read DQ_WIDTH*DQ_BURST (512 as of now) bits + // from DRAM, and have to pipe 256 bit partitions of + // it to the PCI. We may read data each cycle from + // DRAM and have to buffer some of those. + wire rbf_empty, rbf_rd_valid, fifo_almost_full, fifo_valid; + (*KEEP = "TRUE"*) wire rbf_full; + (*KEEP = "TRUE"*) reg [19:0] dbg_rd_ctr; + rdback_fifo rbf( + .full(rbf_full), + .prog_full(fifo_almost_full), + .empty(rbf_empty), + .wr_en(mode_r == READ_MODE ? rd_valid && ~ignore_read_r: dsr_valid), + // shuffle data because fifo outputs them on wrong order + .din(mode_r == READ_MODE ? {rd_data[255:0],rd_data[511:256]} : {dsr_out[255:0],dsr_out[511:256]}), + .rd_en(c2h_tready_0), + .dout(c2h_tdata_0), + .valid(fifo_valid), + .clk(clk), + .srst(rst) + ); + + reg proc_flush_ns, proc_flush_r; + // we count the remaining space in terms of + // AXI transactions + // e.g. 1024 reads will take up 2048 + reg [11:0] buffer_space_ns, buffer_space_r; + + always @* begin + tlast = `LOW; + ignore_read_ns = ignore_read_r; + ignore_flush_ns = ignore_flush_r; + buffer_space_ns = buffer_space_r; + if(per_rd_init || per_zq_init || per_ref_init) begin + ignore_read_ns = per_rd_init; + ignore_flush_ns = `HIGH; + end + if(rd_valid_r) + ignore_read_ns = `LOW; + proc_flush_ns = proc_flush_r; + if(flush) begin + if(ignore_flush_r) + ignore_flush_ns = `LOW; + else + proc_flush_ns = `HIGH; + end + mode_ns = mode_r; + if(switch_mode) + mode_ns = ~mode_r; + if(&xctr_r && (c2h_tready_0 && c2h_tvalid_0)) begin + tlast = `HIGH; + end + // Send what's remaining in the fifo + // to host with a random length transfer + // (tlast is not based on the counter value) + if(proc_flush_r) begin + if(c2h_tready_0 && rbf_empty && ~dsr_valid) begin + tlast = `HIGH; + proc_flush_ns = `LOW; + end + else + proc_flush_ns = `HIGH; + end + if(read_seq_incoming) begin + if(c2h_tvalid_0 && c2h_tready_0) begin + buffer_space_ns = (buffer_space_r - (incoming_reads << 1)) + 1; + end + else begin + buffer_space_ns = (buffer_space_r - (incoming_reads << 1)); + end + end + else begin + if(c2h_tvalid_0 && c2h_tready_0) begin + if(~(proc_flush_r && rbf_empty && ~dsr_valid)) + buffer_space_ns = buffer_space_r + 1; + end + end + end + + always @(posedge clk) begin + if(rst) begin + dbg_rd_ctr <= 20'b0; + xctr_r <= 15'b0; + proc_flush_r <= `LOW; + mode_r <= READ_MODE; + ignore_read_r <= 1'b0; + ignore_flush_r <= 1'b0; + rd_valid_r <= 1'b0; + buffer_space_r <= 12'd2048; + end + else begin + if(rd_valid && ~ignore_read_r && ~rbf_full) + dbg_rd_ctr <= dbg_rd_ctr + 1'b1; + else + dbg_rd_ctr <= dbg_rd_ctr; + buffer_space_r <= buffer_space_ns; + mode_r <= mode_ns; + rd_valid_r <= rd_valid; + ignore_read_r <= ignore_read_ns; + ignore_flush_r <= ignore_flush_ns; + if(proc_flush_r && tlast) + xctr_r <= 15'b0; + else if(c2h_tready_0 && c2h_tvalid_0) begin + xctr_r <= xctr_r + 1; + end + proc_flush_r <= proc_flush_ns; + end + end + + assign c2h_tkeep_0 = {(`XDMA_AXI_DATA_WIDTH/8){1'b1}}; + assign c2h_tlast_0 = tlast; + assign c2h_tvalid_0 = proc_flush_r && rbf_empty && ~dsr_valid ? `HIGH : fifo_valid; + + assign buffer_space = buffer_space_r >> 1; + +endmodule |