From bff92fcf9155f6589b36b3a696d8d1cdb7129d52 Mon Sep 17 00:00:00 2001 From: "Aleksi Korsman aleksi.korsman@aalto.fi" Date: Fri, 22 Nov 2024 16:39:47 +0100 Subject: [PATCH 1/7] Add skeleton for atomic manager --- src/ccu/ace_ccu_atomic_manager.sv | 49 ++++++++++++++++++++++++ src/ccu/ace_ccu_master_path.sv | 5 ++- src/ccu/ace_ccu_snoop_interconnect.sv | 54 +++++++++++++++++++++++---- src/ccu/ace_ccu_snoop_path.sv | 9 +++-- src/ccu/ace_ccu_snoop_req.sv | 8 +++- src/ccu/ace_ccu_top.sv | 32 ++++++++++++++-- src/ccu/ccu_ctrl_r_snoop.sv | 7 +++- src/ccu/ccu_ctrl_wr_snoop.sv | 7 +++- 8 files changed, 152 insertions(+), 19 deletions(-) create mode 100644 src/ccu/ace_ccu_atomic_manager.sv diff --git a/src/ccu/ace_ccu_atomic_manager.sv b/src/ccu/ace_ccu_atomic_manager.sv new file mode 100644 index 0000000..9d5bb88 --- /dev/null +++ b/src/ccu/ace_ccu_atomic_manager.sv @@ -0,0 +1,49 @@ +// Manager for monitoring Exclusive sequences +// Similar to the PoS monitor defined in the ACE spec +module ace_ccu_atomic_manager #( + parameter int unsigned AmIdxWidth = 0, + parameter type mst_idx_t = logic, + + localparam type am_idx_t = logic [AmIdxWidth-1:0] +) ( + input logic clk_i, + input logic rst_ni, + + input logic am_ex_req_i, + /// 0 - Exclusive Load, 1 - Exclusive Store + input logic am_ex_type_i, + input am_idx_t am_ex_addr_i, + /// Master ID + input mst_idx_t am_ex_id_i, + output logic am_ex_okay_o, +); + +localparam int unsigned NumEntries = 2**CmIdxWidth; + +mst_idx_t id_mask_q, id_mask_d; + +always_comb begin + id_mask_d = id_mask_q; + am_ex_okay_o = 1'b0; + if (am_req_i) begin + if (am_ex_type_i) begin + // Exclusive Store + logic new_mask = id_mask_q[am_ex_addr_i] & am_ex_id_i; + id_mask_d[am_ex_addr_i] = new_mask; + am_ex_okay_o = !(new_mask == 0); + end else begin + // Exclusive Load + id_mask_d[am_ex_addr_i] = id_mask_q[am_ex_addr_i] | am_ex_id_i; + end + end +end + +always_ff @(posedge clk_i or negedge rst_ni) begin + if (!rst_ni) begin + id_mask_q <= '0; + end else begin + id_mask_q <= id_mask_d; + end +end + +endmodule diff --git a/src/ccu/ace_ccu_master_path.sv b/src/ccu/ace_ccu_master_path.sv index 2f249fe..a7107fd 100644 --- a/src/ccu/ace_ccu_master_path.sv +++ b/src/ccu/ace_ccu_master_path.sv @@ -34,6 +34,7 @@ module ace_ccu_master_path import ace_pkg::*; parameter type snoop_resp_t = logic, parameter type domain_mask_t = logic, parameter type domain_set_t = logic, + parameter type mst_idx_t = logic, // Local parameters localparam int unsigned NoGroups = NoSlvPorts / NoSlvPerGroup, localparam int unsigned NoSnoopPortsPerGroup = 2, @@ -47,6 +48,7 @@ module ace_ccu_master_path import ace_pkg::*; output slv_resp_t [NoSlvPorts-1:0] slv_resp_o, output snoop_req_t [NoSnoopPorts-1:0] snoop_req_o, output domain_mask_t[NoSnoopPorts-1:0] snoop_masks_o, + output mst_idx_t [NoSnoopPorts-1:0] snoop_idx_o, input snoop_resp_t [NoSnoopPorts-1:0] snoop_resp_i, output mst_req_t mst_req_o, input mst_resp_t mst_resp_i, @@ -387,7 +389,8 @@ module ace_ccu_master_path import ace_pkg::*; .domain_set_i (domain_set_i [(NoSlvPerGroup*i)+:NoSlvPerGroup]), .snoop_reqs_o (snoop_req_o [(2*i)+:2]), .snoop_resps_i (snoop_resp_i [(2*i)+:2]), - .snoop_masks_o (snoop_masks_o [(2*i)+:2]) + .snoop_masks_o (snoop_masks_o [(2*i)+:2]), + .snoop_idx_o (snoop_idx_o [(2*i)+:2]) ); for (genvar j = 1; j < NoMemPortsPerGroup; j++) begin : gen_ace_to_axi diff --git a/src/ccu/ace_ccu_snoop_interconnect.sv b/src/ccu/ace_ccu_snoop_interconnect.sv index 36fc22a..b1a50b5 100644 --- a/src/ccu/ace_ccu_snoop_interconnect.sv +++ b/src/ccu/ace_ccu_snoop_interconnect.sv @@ -8,20 +8,26 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( parameter bit BufferInpResp = 1, parameter int unsigned CmAddrWidth = 0, parameter int unsigned CmAddrBase = 0, + parameter int unsigned AmAddrWidth = 0, + parameter int unsigned AmAddrBase = 0, + parameter type NumMasters = 0, parameter type ac_chan_t = logic, parameter type cr_chan_t = logic, parameter type cd_chan_t = logic, parameter type snoop_req_t = logic, parameter type snoop_resp_t = logic, + parameter type mst_idx_t = logic, localparam type oup_sel_t = logic [NumOup-1:0], - localparam type cm_addr_t = logic [CmAddrWidth-1:0] + localparam type cm_addr_t = logic [CmAddrWidth-1:0], + localparam type am_addr_t = logic [AmAddrWidth-1:0] ) ( input logic clk_i, input logic rst_ni, input oup_sel_t [NumInp-1:0] inp_sel_i, + input mst_idx_t [NumInp-1:0] inp_idx_i, input snoop_req_t [NumInp-1:0] inp_req_i, output snoop_resp_t [NumInp-1:0] inp_resp_o, output snoop_req_t [NumOup-1:0] oup_req_o, @@ -30,7 +36,15 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( output logic cm_valid_o, output logic cm_ready_o, output cm_addr_t cm_addr_o, - input logic cm_stall_i + input logic cm_stall_i, + output logic am_ex_valid_o, + output logic am_ex_ready_o, + output am_addr_t am_ex_addr_o, + /// 0 - Exclusive Load, 1 - Exclusive Store + output logic am_ex_type_o, + /// One-hot encoding of the initiator + output oup_sel_t am_ex_id_o, + input logic am_ex_okay_i ); typedef logic [$clog2(NumInp)-1:0] inp_idx_t; @@ -38,6 +52,7 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( typedef struct packed { oup_sel_t sel; inp_idx_t idx; + logic ex_okay; } ctrl_t; logic [NumInp-1:0] inp_ac_valids, inp_ac_readies; @@ -48,6 +63,7 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( cd_chan_t [NumInp-1:0] inp_cd_chans; oup_sel_t [NumInp-1:0] inp_sel; + mst_idx_t [NumInp-1:0] inp_idx; logic [NumOup-1:0] oup_ac_valids, oup_ac_readies; @@ -60,6 +76,9 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( logic ac_valid, ac_ready; logic oup_ac_valid, oup_ac_ready; ac_chan_t ac_chan; + mst_idx_t ac_mst_idx; + am_addr_t am_ex_addr; + logic ex_okay; logic req_ctrl_valid, req_ctrl_ready; logic resp_ctrl_valid, resp_ctrl_ready; @@ -77,15 +96,18 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( typedef struct packed { ac_chan_t ac; oup_sel_t sel; + mst_idx_t idx; } ac_fifo_entry_t; ac_fifo_entry_t ac_fifo_in, ac_fifo_out; assign ac_fifo_in.ac = inp_req_i[i].ac; assign ac_fifo_in.sel = inp_sel_i[i]; + assign ac_fifo_in.idx = inp_idx_i[i]; assign inp_ac_chans[i] = ac_fifo_out.ac; assign inp_sel[i] = ac_fifo_out.sel; + assign inp_idx[i] = ac_fifo_out.idx; stream_fifo_optimal_wrap #( .Depth (2), @@ -108,6 +130,7 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( assign inp_resp_o[i].ac_ready = inp_ac_readies[i]; assign inp_ac_chans[i] = inp_req_i[i].ac; assign inp_sel[i] = inp_sel_i[i]; + assign inp_idx[i] = inp_idx_i[i]; end if (BufferInpResp) begin : gen_buffer_resp stream_fifo_optimal_wrap #( @@ -233,16 +256,33 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( .ac_valid_o (ac_valid), .ac_ready_i (ac_ready), .ac_chan_o (ac_chan), + .ac_mst_idx_o (ac_mst_idx), .ctrl_valid_o (req_ctrl_valid), .ctrl_ready_i (req_ctrl_ready), .ctrl_o (req_ctrl) ); - assign cm_valid_o = ac_valid; - assign cm_ready_o = oup_ac_ready; - assign cm_addr_o = ac_chan.addr[CmAddrBase+:CmAddrWidth]; - assign ac_ready = oup_ac_ready && !cm_stall_i; - assign oup_ac_valid = ac_valid && !cm_stall_i; + ace_ccu_atomic_manager #( + .AmIdxWidth(CmAddrWidth), + .mst_idx_t (mst_idx_t) + ) i_atomic_manager ( + .clk_i, + .rst_ni, + .am_ex_req_i (ac_valid && ac_atomic), + .am_ex_type_i (0), // TODO + .am_ex_addr_i (am_ex_addr), + .am_ex_id_i (am_ex_id), + .am_ex_okay_o (ex_okay) + ); + + assign am_ex_id_o = ac_mst_idx; + assign am_ex_addr = ac_chan.addr[AmAddrBase+:AmAddrWidth]; + assign cm_valid_o = ac_valid; + assign cm_ready_o = oup_ac_ready; + assign cm_addr_o = ac_chan.addr[CmAddrBase+:CmAddrWidth]; + assign ac_ready = oup_ac_ready && !cm_stall_i; + assign oup_ac_valid = ac_valid && !cm_stall_i; + assign req_ctrl.ex_okay = ex_okay; stream_fifo_optimal_wrap #( .Depth (2), diff --git a/src/ccu/ace_ccu_snoop_path.sv b/src/ccu/ace_ccu_snoop_path.sv index bc07d99..ae7024b 100644 --- a/src/ccu/ace_ccu_snoop_path.sv +++ b/src/ccu/ace_ccu_snoop_path.sv @@ -30,7 +30,8 @@ module ace_ccu_snoop_path import ace_pkg::*; import ccu_pkg::*; #( input domain_set_t [NoRules-1:0] domain_set_i, output snoop_req_t [1:0] snoop_reqs_o, input snoop_resp_t [1:0] snoop_resps_i, - output domain_mask_t [1:0] snoop_masks_o + output domain_mask_t [1:0] snoop_masks_o, + output domain_mask_t [1:0] snoop_idx_o ); localparam RuleIdBits = $clog2(NoRules); @@ -105,7 +106,8 @@ module ace_ccu_snoop_path import ace_pkg::*; import ccu_pkg::*; #( .snoop_req_o (snoop_reqs_o [0]), .snoop_resp_i (snoop_resps_i [0]), .domain_set_i (domain_set_i[write_rule_idx]), - .domain_mask_o (snoop_masks_o [0]) + .domain_mask_o (snoop_masks_o [0]), + .mst_idx_o (snoop_idx_o [0]) ); /////////////// @@ -153,7 +155,8 @@ module ace_ccu_snoop_path import ace_pkg::*; import ccu_pkg::*; #( .snoop_req_o (snoop_reqs_o [1]), .snoop_resp_i (snoop_resps_i [1]), .domain_set_i (domain_set_i[read_rule_idx]), - .domain_mask_o (snoop_masks_o [1]) + .domain_mask_o (snoop_masks_o [1]), + .mst_idx_o (snoop_idx_o [1]) ); endmodule diff --git a/src/ccu/ace_ccu_snoop_req.sv b/src/ccu/ace_ccu_snoop_req.sv index 647738a..cb43f10 100644 --- a/src/ccu/ace_ccu_snoop_req.sv +++ b/src/ccu/ace_ccu_snoop_req.sv @@ -2,7 +2,8 @@ module ace_ccu_snoop_req #( parameter int unsigned NumInp = 0, parameter int unsigned NumOup = 0, parameter type ac_chan_t = logic, - parameter type ctrl_t = logic + parameter type ctrl_t = logic, + parameter type mst_idx_t = logic ) ( input logic clk_i, @@ -11,9 +12,11 @@ module ace_ccu_snoop_req #( output logic [NumInp-1:0] ac_readies_o, input ac_chan_t [NumInp-1:0] ac_chans_i, input logic [NumInp-1:0][NumOup-1:0] ac_sel_i, + input mst_idx_t [NumInp-1:0] ac_mst_idxs_i, output logic ac_valid_o, input logic ac_ready_i, output ac_chan_t ac_chan_o, + output mst_idx_t ac_mst_idx_o, output logic ctrl_valid_o, input logic ctrl_ready_i, output ctrl_t ctrl_o @@ -44,7 +47,8 @@ rr_arb_tree #( .idx_o (ac_idx) ); -assign ac_sel = ac_sel_i[ac_idx]; +assign ac_sel = ac_sel_i [ac_idx]; +assign ac_mst_idx_o = ac_mst_idxs_i[ac_idx]; stream_fork #( .N_OUP (2) diff --git a/src/ccu/ace_ccu_top.sv b/src/ccu/ace_ccu_top.sv index 8aa110c..4d4be3f 100644 --- a/src/ccu/ace_ccu_top.sv +++ b/src/ccu/ace_ccu_top.sv @@ -13,6 +13,7 @@ module ace_ccu_top import ace_pkg::*; parameter int unsigned DcacheLineWidth = 0, parameter int unsigned CmAddrBase = $clog2(DcacheLineWidth >> 3), parameter int unsigned CmAddrWidth = 8, + parameter int unsigned AmAddrWidth = 8, parameter type slv_ar_chan_t = logic, parameter type slv_aw_chan_t = logic, parameter type slv_b_chan_t = logic, @@ -32,7 +33,8 @@ module ace_ccu_top import ace_pkg::*; parameter type snoop_req_t = logic, parameter type snoop_resp_t = logic, parameter type domain_mask_t = `DOMAIN_MASK_T(NoSlvPorts), - parameter type domain_set_t = `DOMAIN_SET_T + parameter type domain_set_t = `DOMAIN_SET_T, + parameter type mst_idx_t = domain_mask_t ) ( input logic clk_i, input logic rst_ni, @@ -46,6 +48,7 @@ module ace_ccu_top import ace_pkg::*; ); typedef logic [CmAddrWidth-1:0] cm_idx_t; + typedef logic [AmAddrWidth-1:0] am_idx_t; // Parameters to be used for testing/debugging // Otherwise assume they are hardcoded @@ -58,6 +61,7 @@ module ace_ccu_top import ace_pkg::*; // To snoop interconnect domain_mask_t [NoSnoopPorts-1:0] snoop_sel; + mst_idx_t [NoSnoopPorts-1:0] snoop_idx; snoop_req_t [NoSnoopPorts-1:0] snoop_reqs; snoop_resp_t [NoSnoopPorts-1:0] snoop_resps; @@ -69,6 +73,10 @@ module ace_ccu_top import ace_pkg::*; logic cm_snoop_stall; cm_idx_t cm_snoop_addr; + // Atomic management signals + mst_idx_t am_ex_id; + am_idx_t am_ex_addr; + ace_ccu_master_path #( .AxiAddrWidth (AxiAddrWidth), .AxiDataWidth (AxiDataWidth), @@ -98,7 +106,8 @@ module ace_ccu_top import ace_pkg::*; .snoop_req_t (snoop_req_t ), .snoop_resp_t (snoop_resp_t), .domain_set_t (domain_set_t), - .domain_mask_t (domain_mask_t) + .domain_mask_t (domain_mask_t), + .mst_idx_t (domain_mask_t) ) i_master_path ( .clk_i (clk_i), .rst_ni (rst_ni), @@ -110,6 +119,7 @@ module ace_ccu_top import ace_pkg::*; .mst_resp_i (mst_resp_i), .domain_set_i (domain_set_i), .snoop_masks_o (snoop_sel), + .snoop_idx_o (snoop_idx), .cm_req_o (cm_x_req), .cm_addr_o (cm_x_addr) ); @@ -127,11 +137,13 @@ module ace_ccu_top import ace_pkg::*; .cr_chan_t (snoop_cr_t), .cd_chan_t (snoop_cd_t), .snoop_req_t (snoop_req_t), - .snoop_resp_t (snoop_resp_t) + .snoop_resp_t (snoop_resp_t), + .mst_idx_t (domain_mask_t) ) i_snoop_interconnect ( .clk_i (clk_i), .rst_ni (rst_ni), .inp_sel_i (snoop_sel), + .inp_idx_i (snoop_idx), .inp_req_i (snoop_reqs), .inp_resp_o (snoop_resps), .oup_req_o (snoop_req_o), @@ -139,7 +151,8 @@ module ace_ccu_top import ace_pkg::*; .cm_valid_o (cm_snoop_valid), .cm_ready_o (cm_snoop_ready), .cm_addr_o (cm_snoop_addr), - .cm_stall_i (cm_snoop_stall) + .cm_stall_i (cm_snoop_stall), + .am_ex_id_o (am_ex_id) ); ace_ccu_conflict_manager #( @@ -159,6 +172,17 @@ module ace_ccu_top import ace_pkg::*; .cm_x_addr_i (cm_x_addr) ); + ace_ccu_atomic_manager #( + .AmIdxWidth(CmAddrWidth) + ) i_atomic_manager ( + .clk_i, + .rst_ni, + .am_ex_req_i (cm_snoop_valid), + .am_ex_type_i (0), // TODO + .am_ex_addr_i (am_ex_addr), + .am_ex_id_i (am_ex_id) + ); + endmodule module ace_ccu_top_intf #( diff --git a/src/ccu/ccu_ctrl_r_snoop.sv b/src/ccu/ccu_ctrl_r_snoop.sv index 93be154..846bd4a 100644 --- a/src/ccu/ccu_ctrl_r_snoop.sv +++ b/src/ccu/ccu_ctrl_r_snoop.sv @@ -22,6 +22,8 @@ module ccu_ctrl_r_snoop #( parameter type domain_set_t = logic, /// Domain mask type parameter type domain_mask_t = logic, + /// Master index type + parameter type mst_idx_t = logic, /// Fixed value for AXLEN for write back parameter int unsigned AXLEN = 0, /// Fixed value for AXSIZE for write back @@ -51,7 +53,9 @@ module ccu_ctrl_r_snoop #( /// Domain masks set for the current AR initiator input domain_set_t domain_set_i, /// Ax mask to be used for the snoop request - output domain_mask_t domain_mask_o + output domain_mask_t domain_mask_o, + /// One-hot encoding of the index of the cached master + output mst_idx_t mst_idx_o ); // Indices for stream fork dynamic mask @@ -338,6 +342,7 @@ stream_fork_dynamic #( // Note: this signal should flow along with AC always_comb begin domain_mask_o = '0; + mst_idx_o = domain_set_i.initiator; case (slv_req_i.ar.domain) NonShareable: domain_mask_o = 0; InnerShareable: domain_mask_o = domain_set_i.inner; diff --git a/src/ccu/ccu_ctrl_wr_snoop.sv b/src/ccu/ccu_ctrl_wr_snoop.sv index c120b2c..1879d9c 100644 --- a/src/ccu/ccu_ctrl_wr_snoop.sv +++ b/src/ccu/ccu_ctrl_wr_snoop.sv @@ -22,6 +22,8 @@ module ccu_ctrl_wr_snoop #( parameter type domain_set_t = logic, /// Domain mask type parameter type domain_mask_t = logic, + /// Master index type + parameter type mst_idx_t = logic, /// Fixed value for AXLEN for write back parameter int unsigned AXLEN = 0, /// Fixed value for AXSIZE for write back @@ -51,7 +53,9 @@ module ccu_ctrl_wr_snoop #( /// Domain masks set for the current AW initiator input domain_set_t domain_set_i, /// Ax mask to be used for the snoop request - output domain_mask_t domain_mask_o + output domain_mask_t domain_mask_o, + /// One-hot encoding of the index of the cached master + output mst_idx_t mst_idx_o ); // Data structure to store AW request and decoded snoop transaction @@ -229,6 +233,7 @@ end // Note: this signal should flow along with AC always_comb begin domain_mask_o = '0; + mst_idx_o = domain_set_i.initiator; case (slv_req_i.aw.domain) NonShareable: domain_mask_o = 0; InnerShareable: domain_mask_o = domain_set_i.inner; From dbe40150929a1b3396ce6620df24a1205974a290 Mon Sep 17 00:00:00 2001 From: "Aleksi Korsman aleksi.korsman@aalto.fi" Date: Mon, 25 Nov 2024 11:04:04 +0100 Subject: [PATCH 2/7] Add decoding for lock --- src/ace_ar_transaction_decoder.sv | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ace_ar_transaction_decoder.sv b/src/ace_ar_transaction_decoder.sv index 4ad385f..98a1735 100644 --- a/src/ace_ar_transaction_decoder.sv +++ b/src/ace_ar_transaction_decoder.sv @@ -12,6 +12,8 @@ module ace_ar_transaction_decoder import ace_pkg::*; #( arsnoop_t arsnoop; +logic lock; + logic is_shareable; logic is_system; logic is_barrier; @@ -32,6 +34,7 @@ logic dvm_complete; logic dvm_message; assign arsnoop = ar_i.snoop; +assign lock = ar_i.lock; assign is_shareable = ar_i.domain inside {InnerShareable, OuterShareable}; assign is_system = ar_i.domain inside {System}; @@ -59,6 +62,8 @@ always_comb begin snoop_info_o.accepts_dirty = 1'b0; snoop_info_o.accepts_dirty_shared = 1'b0; snoop_info_o.accepts_shared = 1'b0; + snoop_info_o.excl_load = 1'b0; + snoop_info_o.excl_store = 1'b0; unique case (1'b1) read_no_snoop: begin snooping_o = 1'b0; @@ -70,9 +75,11 @@ always_comb begin snoop_info_o.accepts_dirty = 1'b1; snoop_info_o.accepts_dirty_shared = 1'b1; snoop_info_o.accepts_shared = 1'b1; + snoop_info_o.excl_load = lock; end read_clean: begin snoop_info_o.accepts_shared = 1'b1; + snoop_info_o.excl_load = lock; end read_not_shared_dirty: begin snoop_info_o.accepts_dirty = 1'b1; @@ -82,7 +89,8 @@ always_comb begin snoop_info_o.accepts_dirty = 1'b1; end clean_unique: begin - snoop_info_o.snoop_trs = acsnoop_t'(CleanInvalid); + snoop_info_o.snoop_trs = acsnoop_t'(CleanInvalid); + snoop_info_o.excl_store = lock; end make_unique: begin snoop_info_o.snoop_trs = acsnoop_t'(MakeInvalid); From bfd109b51fb6d355e1473fe20cc1e5ab0d0b3ebe Mon Sep 17 00:00:00 2001 From: "Aleksi Korsman aleksi.korsman@aalto.fi" Date: Mon, 25 Nov 2024 11:04:24 +0100 Subject: [PATCH 3/7] Add excl info to snoop_info --- src/ace_pkg.sv | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ace_pkg.sv b/src/ace_pkg.sv index f27f526..4f9feeb 100644 --- a/src/ace_pkg.sv +++ b/src/ace_pkg.sv @@ -44,6 +44,8 @@ package ace_pkg; logic accepts_dirty; logic accepts_dirty_shared; logic accepts_shared; + logic excl_load; + logic excl_store; } snoop_info_t; /////////////// From 2512a7ffc68795a58d9d557da611c9c091d4a4a0 Mon Sep 17 00:00:00 2001 From: "Aleksi Korsman aleksi.korsman@aalto.fi" Date: Mon, 25 Nov 2024 11:05:26 +0100 Subject: [PATCH 4/7] Propagate exclusive signals --- src/ccu/ace_ccu_atomic_manager.sv | 30 +++++----- src/ccu/ace_ccu_master_path.sv | 11 +++- src/ccu/ace_ccu_snoop_interconnect.sv | 79 ++++++++++++++++----------- src/ccu/ace_ccu_snoop_path.sv | 17 +++++- src/ccu/ace_ccu_snoop_req.sv | 6 ++ src/ccu/ace_ccu_top.sv | 19 +++---- src/ccu/ccu_ctrl_r_snoop.sv | 24 ++++++-- 7 files changed, 114 insertions(+), 72 deletions(-) diff --git a/src/ccu/ace_ccu_atomic_manager.sv b/src/ccu/ace_ccu_atomic_manager.sv index 9d5bb88..34df3a0 100644 --- a/src/ccu/ace_ccu_atomic_manager.sv +++ b/src/ccu/ace_ccu_atomic_manager.sv @@ -9,13 +9,12 @@ module ace_ccu_atomic_manager #( input logic clk_i, input logic rst_ni, - input logic am_ex_req_i, - /// 0 - Exclusive Load, 1 - Exclusive Store - input logic am_ex_type_i, - input am_idx_t am_ex_addr_i, + input logic am_ex_load_i, + input logic am_ex_store_i, + input am_idx_t am_ex_addr_i, /// Master ID - input mst_idx_t am_ex_id_i, - output logic am_ex_okay_o, + input mst_idx_t am_ex_id_i, + output logic am_ex_okay_o, ); localparam int unsigned NumEntries = 2**CmIdxWidth; @@ -25,16 +24,15 @@ mst_idx_t id_mask_q, id_mask_d; always_comb begin id_mask_d = id_mask_q; am_ex_okay_o = 1'b0; - if (am_req_i) begin - if (am_ex_type_i) begin - // Exclusive Store - logic new_mask = id_mask_q[am_ex_addr_i] & am_ex_id_i; - id_mask_d[am_ex_addr_i] = new_mask; - am_ex_okay_o = !(new_mask == 0); - end else begin - // Exclusive Load - id_mask_d[am_ex_addr_i] = id_mask_q[am_ex_addr_i] | am_ex_id_i; - end + if (am_ex_store_i) begin + // Exclusive Store + mst_idx_t new_mask = id_mask_q[am_ex_addr_i] & am_ex_id_i; + id_mask_d[am_ex_addr_i] = new_mask; + am_ex_okay_o = !(new_mask == 0); + end else if (am_ex_load_i) begin + // Exclusive Load + id_mask_d[am_ex_addr_i] = id_mask_q[am_ex_addr_i] | am_ex_id_i; + am_ex_okay_o = 1'b1; end end diff --git a/src/ccu/ace_ccu_master_path.sv b/src/ccu/ace_ccu_master_path.sv index a7107fd..74fae62 100644 --- a/src/ccu/ace_ccu_master_path.sv +++ b/src/ccu/ace_ccu_master_path.sv @@ -54,7 +54,11 @@ module ace_ccu_master_path import ace_pkg::*; input mst_resp_t mst_resp_i, output logic [2*NoGroups-1:0] cm_req_o, - output cm_addr_t [2*NoGroups-1:0] cm_addr_o + output cm_addr_t [2*NoGroups-1:0] cm_addr_o, + + output logic [2*NoGroups-1:0] excl_load_o, + output logic [2*NoGroups-1:0] excl_store_o, + input logic [2*NoGroups-1:0] excl_resp_i ); typedef logic [AxiAddrWidth -1:0] addr_t; @@ -390,7 +394,10 @@ module ace_ccu_master_path import ace_pkg::*; .snoop_reqs_o (snoop_req_o [(2*i)+:2]), .snoop_resps_i (snoop_resp_i [(2*i)+:2]), .snoop_masks_o (snoop_masks_o [(2*i)+:2]), - .snoop_idx_o (snoop_idx_o [(2*i)+:2]) + .snoop_idx_o (snoop_idx_o [(2*i)+:2]), + .excl_load_o (excl_load_o [(2*i)+:2]), + .excl_store_o (excl_store_o [(2*i)+:2]), + .excl_resp_i (excl_resp_i [(2*i)+:2]) ); for (genvar j = 1; j < NoMemPortsPerGroup; j++) begin : gen_ace_to_axi diff --git a/src/ccu/ace_ccu_snoop_interconnect.sv b/src/ccu/ace_ccu_snoop_interconnect.sv index b1a50b5..5826adf 100644 --- a/src/ccu/ace_ccu_snoop_interconnect.sv +++ b/src/ccu/ace_ccu_snoop_interconnect.sv @@ -33,18 +33,16 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( output snoop_req_t [NumOup-1:0] oup_req_o, input snoop_resp_t [NumOup-1:0] oup_resp_i, + input logic [NumInp-1:0] excl_load_i, + input logic [NumInp-1:0] excl_store_i, + output logic [NumInp-1:0] excl_resp_o, + output logic cm_valid_o, output logic cm_ready_o, output cm_addr_t cm_addr_o, input logic cm_stall_i, - output logic am_ex_valid_o, - output logic am_ex_ready_o, - output am_addr_t am_ex_addr_o, - /// 0 - Exclusive Load, 1 - Exclusive Store - output logic am_ex_type_o, - /// One-hot encoding of the initiator - output oup_sel_t am_ex_id_o, - input logic am_ex_okay_i + + input logic [NumInp/2-1:0] ); typedef logic [$clog2(NumInp)-1:0] inp_idx_t; @@ -52,7 +50,7 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( typedef struct packed { oup_sel_t sel; inp_idx_t idx; - logic ex_okay; + logic excl_okay; } ctrl_t; logic [NumInp-1:0] inp_ac_valids, inp_ac_readies; @@ -61,6 +59,7 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( cr_chan_t [NumInp-1:0] inp_cr_chans; logic [NumInp-1:0] inp_cd_valids, inp_cd_readies; cd_chan_t [NumInp-1:0] inp_cd_chans; + logic [NumInp-1:0] inp_excl_load, inp_excl_store; oup_sel_t [NumInp-1:0] inp_sel; mst_idx_t [NumInp-1:0] inp_idx; @@ -75,10 +74,10 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( logic ac_valid, ac_ready; logic oup_ac_valid, oup_ac_ready; + logic oup_excl_load, oup_excl_store; ac_chan_t ac_chan; mst_idx_t ac_mst_idx; am_addr_t am_ex_addr; - logic ex_okay; logic req_ctrl_valid, req_ctrl_ready; logic resp_ctrl_valid, resp_ctrl_ready; @@ -97,17 +96,23 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( ac_chan_t ac; oup_sel_t sel; mst_idx_t idx; + logic excl_load; + logic excl_store; } ac_fifo_entry_t; ac_fifo_entry_t ac_fifo_in, ac_fifo_out; - assign ac_fifo_in.ac = inp_req_i[i].ac; - assign ac_fifo_in.sel = inp_sel_i[i]; - assign ac_fifo_in.idx = inp_idx_i[i]; + assign ac_fifo_in.ac = inp_req_i[i].ac; + assign ac_fifo_in.sel = inp_sel_i[i]; + assign ac_fifo_in.idx = inp_idx_i[i]; + assign ac_fifo_in.excl_load = excl_load_i[i]; + assign ac_fifo_in.excl_store = excl_store_i[i]; - assign inp_ac_chans[i] = ac_fifo_out.ac; - assign inp_sel[i] = ac_fifo_out.sel; - assign inp_idx[i] = ac_fifo_out.idx; + assign inp_ac_chans[i] = ac_fifo_out.ac; + assign inp_sel[i] = ac_fifo_out.sel; + assign inp_idx[i] = ac_fifo_out.idx; + assign inp_excl_load[i] = ac_fifo_out.excl_load; + assign inp_excl_store[i] = ac_fifo_out.excl_store; stream_fifo_optimal_wrap #( .Depth (2), @@ -131,6 +136,8 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( assign inp_ac_chans[i] = inp_req_i[i].ac; assign inp_sel[i] = inp_sel_i[i]; assign inp_idx[i] = inp_idx_i[i]; + assign inp_excl_load[i] = excl_load_i[i]; + assign inp_excl_store[i] = excl_store_i[i]; end if (BufferInpResp) begin : gen_buffer_resp stream_fifo_optimal_wrap #( @@ -144,10 +151,10 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( .usage_o (), .valid_i (inp_cr_valids [i]), .ready_o (inp_cr_readies[i]), - .data_i (inp_cr_chans [i]), + .data_i ({inp_cr_chans [i], cr_ctrl.excl_okay}), .valid_o (inp_resp_o[i].cr_valid), .ready_i (inp_req_i [i].cr_ready), - .data_o (inp_resp_o[i].cr_resp) + .data_o ({inp_resp_o[i].cr_resp, excl_resp_o[i]}) ); stream_fifo_optimal_wrap #( .Depth (4), @@ -172,6 +179,7 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( assign inp_cd_readies[i] = inp_req_i[i].cd_ready; assign inp_resp_o[i].cd_valid = inp_cd_valids[i]; assign inp_resp_o[i].cd = inp_cd_chans[i]; + assign excl_resp_o[i] = cr_ctrl.excl_okay; end end @@ -253,36 +261,41 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( .ac_readies_o (inp_ac_readies), .ac_chans_i (inp_ac_chans), .ac_sel_i (inp_sel), + .excl_load_i (inp_excl_load), + .excl_store_i (inp_excl_store), .ac_valid_o (ac_valid), .ac_ready_i (ac_ready), .ac_chan_o (ac_chan), .ac_mst_idx_o (ac_mst_idx), + .excl_load_o (oup_excl_load), + .excl_store_o (oup_excl_store), .ctrl_valid_o (req_ctrl_valid), .ctrl_ready_i (req_ctrl_ready), .ctrl_o (req_ctrl) ); ace_ccu_atomic_manager #( - .AmIdxWidth(CmAddrWidth), - .mst_idx_t (mst_idx_t) + .AmIdxWidth (AmAddrWidth), + .mst_idx_t (mst_idx_t) ) i_atomic_manager ( .clk_i, .rst_ni, - .am_ex_req_i (ac_valid && ac_atomic), - .am_ex_type_i (0), // TODO - .am_ex_addr_i (am_ex_addr), - .am_ex_id_i (am_ex_id), - .am_ex_okay_o (ex_okay) + .am_ex_load_i (am_ex_load), + .am_ex_store_i (am_ex_store), + .am_ex_addr_i (am_ex_addr), + .am_ex_id_i (ac_mst_idx), + .am_ex_okay_o (excl_okay) ); - assign am_ex_id_o = ac_mst_idx; - assign am_ex_addr = ac_chan.addr[AmAddrBase+:AmAddrWidth]; - assign cm_valid_o = ac_valid; - assign cm_ready_o = oup_ac_ready; - assign cm_addr_o = ac_chan.addr[CmAddrBase+:CmAddrWidth]; - assign ac_ready = oup_ac_ready && !cm_stall_i; - assign oup_ac_valid = ac_valid && !cm_stall_i; - assign req_ctrl.ex_okay = ex_okay; + assign am_ex_addr = ac_chan.addr[AmAddrBase+:AmAddrWidth]; + assign am_ex_load = ac_valid && oup_excl_load; + assign am_ex_store = ac_valid && oup_excl_store; + assign cm_valid_o = ac_valid; + assign cm_ready_o = oup_ac_ready; + assign cm_addr_o = ac_chan.addr[CmAddrBase+:CmAddrWidth]; + assign ac_ready = oup_ac_ready && !cm_stall_i; + assign oup_ac_valid = ac_valid && !cm_stall_i; + assign req_ctrl.excl_okay = excl_okay; stream_fifo_optimal_wrap #( .Depth (2), diff --git a/src/ccu/ace_ccu_snoop_path.sv b/src/ccu/ace_ccu_snoop_path.sv index ae7024b..98ccba7 100644 --- a/src/ccu/ace_ccu_snoop_path.sv +++ b/src/ccu/ace_ccu_snoop_path.sv @@ -19,7 +19,8 @@ module ace_ccu_snoop_path import ace_pkg::*; import ccu_pkg::*; #( parameter type snoop_req_t = logic, // Snoop port request type parameter type snoop_resp_t = logic, // Snoop port response type parameter type domain_mask_t = logic, - parameter type domain_set_t = logic + parameter type domain_set_t = logic, + parameter type mst_idx_t = logic ) ( input logic clk_i, input logic rst_ni, @@ -31,7 +32,10 @@ module ace_ccu_snoop_path import ace_pkg::*; import ccu_pkg::*; #( output snoop_req_t [1:0] snoop_reqs_o, input snoop_resp_t [1:0] snoop_resps_i, output domain_mask_t [1:0] snoop_masks_o, - output domain_mask_t [1:0] snoop_idx_o + output mst_idx_t [1:0] snoop_idx_o, + output logic [1:0] excl_load_o, + output logic [1:0] excl_store_o, + input logic [1:0] excl_resp_i ); localparam RuleIdBits = $clog2(NoRules); @@ -110,6 +114,10 @@ module ace_ccu_snoop_path import ace_pkg::*; import ccu_pkg::*; #( .mst_idx_o (snoop_idx_o [0]) ); + // Write channel cannot generate exclusive accesses + assign excl_load_o[0] = 1'b0; + assign excl_store_o[0] = 1'b0; + /////////////// // READ PATH // /////////////// @@ -156,7 +164,10 @@ module ace_ccu_snoop_path import ace_pkg::*; import ccu_pkg::*; #( .snoop_resp_i (snoop_resps_i [1]), .domain_set_i (domain_set_i[read_rule_idx]), .domain_mask_o (snoop_masks_o [1]), - .mst_idx_o (snoop_idx_o [1]) + .mst_idx_o (snoop_idx_o [1]), + .excl_load_o (excl_load_o [1]), + .excl_store_o (excl_store_o [1]), + .excl_resp_i (excl_resp_i [1]) ); endmodule diff --git a/src/ccu/ace_ccu_snoop_req.sv b/src/ccu/ace_ccu_snoop_req.sv index cb43f10..383fb21 100644 --- a/src/ccu/ace_ccu_snoop_req.sv +++ b/src/ccu/ace_ccu_snoop_req.sv @@ -13,10 +13,14 @@ module ace_ccu_snoop_req #( input ac_chan_t [NumInp-1:0] ac_chans_i, input logic [NumInp-1:0][NumOup-1:0] ac_sel_i, input mst_idx_t [NumInp-1:0] ac_mst_idxs_i, + input logic [NumInp-1:0] excl_load_i, + input logic [NumInp-1:0] excl_store_i, output logic ac_valid_o, input logic ac_ready_i, output ac_chan_t ac_chan_o, output mst_idx_t ac_mst_idx_o, + output logic excl_load_o, + output logic excl_store_o, output logic ctrl_valid_o, input logic ctrl_ready_i, output ctrl_t ctrl_o @@ -49,6 +53,8 @@ rr_arb_tree #( assign ac_sel = ac_sel_i [ac_idx]; assign ac_mst_idx_o = ac_mst_idxs_i[ac_idx]; +assign excl_load_o = excl_load_i [ac_idx]; +assign excl_store_o = excl_store_i [ac_idx]; stream_fork #( .N_OUP (2) diff --git a/src/ccu/ace_ccu_top.sv b/src/ccu/ace_ccu_top.sv index 4d4be3f..1da1278 100644 --- a/src/ccu/ace_ccu_top.sv +++ b/src/ccu/ace_ccu_top.sv @@ -74,6 +74,9 @@ module ace_ccu_top import ace_pkg::*; cm_idx_t cm_snoop_addr; // Atomic management signals + logic [2*NoGroups-1:0] excl_load; + logic [2*NoGroups-1:0] excl_store; + logic [2*NoGroups-1:0] excl_resp; mst_idx_t am_ex_id; am_idx_t am_ex_addr; @@ -121,7 +124,10 @@ module ace_ccu_top import ace_pkg::*; .snoop_masks_o (snoop_sel), .snoop_idx_o (snoop_idx), .cm_req_o (cm_x_req), - .cm_addr_o (cm_x_addr) + .cm_addr_o (cm_x_addr), + .excl_load_o (excl_load), + .excl_store_o (excl_store), + .excl_resp_i (excl_resp) ); ace_ccu_snoop_interconnect #( @@ -172,17 +178,6 @@ module ace_ccu_top import ace_pkg::*; .cm_x_addr_i (cm_x_addr) ); - ace_ccu_atomic_manager #( - .AmIdxWidth(CmAddrWidth) - ) i_atomic_manager ( - .clk_i, - .rst_ni, - .am_ex_req_i (cm_snoop_valid), - .am_ex_type_i (0), // TODO - .am_ex_addr_i (am_ex_addr), - .am_ex_id_i (am_ex_id) - ); - endmodule module ace_ccu_top_intf #( diff --git a/src/ccu/ccu_ctrl_r_snoop.sv b/src/ccu/ccu_ctrl_r_snoop.sv index 846bd4a..278e17f 100644 --- a/src/ccu/ccu_ctrl_r_snoop.sv +++ b/src/ccu/ccu_ctrl_r_snoop.sv @@ -50,6 +50,14 @@ module ccu_ctrl_r_snoop #( input mst_snoop_resp_t snoop_resp_i, /// Request channel towards snoop crossbar output mst_snoop_req_t snoop_req_o, + /// Exclusive load access. Flows with snoop_req_o. + output logic excl_load_o, + /// Exclusive store access. Flows with snoop_req_o. + output logic excl_store_o, + /// Response to exclusive transaction. + /// Should flow with snoop_resp_i.cr_valid. + /// 1 - success, 0 - fail. + input logic excl_resp_i, /// Domain masks set for the current AR initiator input domain_set_t domain_set_i, /// Ax mask to be used for the snoop request @@ -115,7 +123,7 @@ end always_ff @(posedge clk_i, negedge rst_ni) begin if (!rst_ni) begin fsm_state_q <= SNOOP_RESP; - rresp_q[3:2] <= '0; + rresp_q <= '0; cd_mask_q <= '0; aw_valid_q <= '0; ar_valid_q <= '0; @@ -123,7 +131,7 @@ always_ff @(posedge clk_i, negedge rst_ni) begin r_last_q <= '0; end else begin fsm_state_q <= fsm_state_d; - rresp_q[3:2] <= rresp_d[3:2]; + rresp_q <= rresp_d; cd_mask_q <= cd_mask_d; aw_valid_q <= aw_valid_d; ar_valid_q <= ar_valid_d; @@ -138,6 +146,8 @@ always_comb begin snoop_req_o.ac.addr = slv_req.ar.addr; snoop_req_o.ac.snoop = slv_req.snoop_info.snoop_trs; snoop_req_o.ac.prot = slv_req.ar.prot; + excl_load_o = slv_req.snoop_info.excl_load; + excl_store_o = slv_req.snoop_info.excl_store; slv_resp_o.ar_ready = snoop_resp_i.ac_ready && slv_req_fifo_not_full; end @@ -159,12 +169,12 @@ always_comb begin mst_req_o.aw.burst = axi_pkg::BURST_WRAP; mst_req_o.aw.domain = slv_req_holder.ar.domain; mst_req_o.aw.snoop = ace_pkg::WriteBack; - mst_req_o.aw.lock = 1'b0; // TODO + mst_req_o.aw.lock = 1'b0; mst_req_o.aw.cache = axi_pkg::CACHE_MODIFIABLE; mst_req_o.aw.prot = slv_req_holder.ar.prot; mst_req_o.aw.qos = slv_req_holder.ar.qos; mst_req_o.aw.region = slv_req_holder.ar.region; - mst_req_o.aw.atop = '0; // TODO + mst_req_o.aw.atop = '0; mst_req_o.aw.user = slv_req_holder.ar.user; mst_req_o.aw.bar = '0; mst_req_o.aw.awunique = 1'b0; @@ -192,7 +202,7 @@ always_comb begin cd_last_d = cd_last_q; fsm_state_d = fsm_state_q; cd_mask_d = cd_mask_q; - rresp_d[3:2] = rresp_q[3:2]; + rresp_d = rresp_q; arlen_counter_clear = 1'b0; mst_req_o.w_valid = 1'b0; mst_req_o.r_ready = 1'b0; @@ -216,6 +226,8 @@ always_comb begin arlen_counter_clear = 1'b1; snoop_req_o.cr_ready = slv_req_fifo_valid; if (snoop_resp_i.cr_valid) begin + rresp_d[0] = excl_resp_i; + rresp_d[1] = 1'b0; rresp_d[2] = resp_dirty; rresp_d[3] = resp_shared; if (snoop_resp_i.cr_resp.DataTransfer) begin @@ -250,7 +262,7 @@ always_comb begin WRITE_CD: begin mst_req_o.w_valid = cd_fork_valid[MEM_W_IDX] && !aw_valid_q; slv_resp_o.r.data = snoop_resp_i.cd.data; - slv_resp_o.r.resp = {rresp_q[3:2], 2'b0}; // something has to happen to 2 lsb when atomic + slv_resp_o.r.resp = {rresp_q[3:2], 1'b0, excl_store_resp_i}; slv_resp_o.r.last = r_last; slv_resp_o.r_valid = cd_fork_valid[MST_R_IDX] && !r_last_q; arlen_counter_en = r_handshake; From bfc710996b7a28289b0f350280cd5282f5ab187c Mon Sep 17 00:00:00 2001 From: "Aleksi Korsman aleksi.korsman@aalto.fi" Date: Mon, 25 Nov 2024 12:33:17 +0100 Subject: [PATCH 5/7] Add atomic manager to bender --- Bender.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/Bender.yml b/Bender.yml index 6c32f74..4a64f3a 100644 --- a/Bender.yml +++ b/Bender.yml @@ -39,6 +39,7 @@ sources: - src/ccu/ace_ccu_snoop_path.sv - src/ccu/ace_ccu_master_path.sv - src/ccu/ace_ccu_conflict_manager.sv + - src/ccu/ace_ccu_atomic_manager.sv - src/ccu/ace_ccu_top.sv #- src/ccu_ctrl_pkg.sv ## Level 2 From 4d30ed2273fc297ddafbf8b98cedb73843cb23c9 Mon Sep 17 00:00:00 2001 From: "Aleksi Korsman aleksi.korsman@aalto.fi" Date: Mon, 25 Nov 2024 12:33:46 +0100 Subject: [PATCH 6/7] Move atomic manager inside snoop req --- src/ccu/ace_ccu_atomic_manager.sv | 12 +++--- src/ccu/ace_ccu_master_path.sv | 3 +- src/ccu/ace_ccu_snoop_interconnect.sv | 59 +++++++++++---------------- src/ccu/ace_ccu_snoop_path.sv | 2 + src/ccu/ace_ccu_snoop_req.sv | 45 ++++++++++++++------ src/ccu/ace_ccu_top.sv | 16 +++++--- src/ccu/ccu_ctrl_r_snoop.sv | 2 +- 7 files changed, 77 insertions(+), 62 deletions(-) diff --git a/src/ccu/ace_ccu_atomic_manager.sv b/src/ccu/ace_ccu_atomic_manager.sv index 34df3a0..ed37a8a 100644 --- a/src/ccu/ace_ccu_atomic_manager.sv +++ b/src/ccu/ace_ccu_atomic_manager.sv @@ -14,21 +14,21 @@ module ace_ccu_atomic_manager #( input am_idx_t am_ex_addr_i, /// Master ID input mst_idx_t am_ex_id_i, - output logic am_ex_okay_o, + output logic am_ex_okay_o ); -localparam int unsigned NumEntries = 2**CmIdxWidth; +localparam int unsigned NumEntries = 2**AmIdxWidth; -mst_idx_t id_mask_q, id_mask_d; +mst_idx_t [NumEntries-1:0] id_mask_q, id_mask_d; always_comb begin id_mask_d = id_mask_q; am_ex_okay_o = 1'b0; if (am_ex_store_i) begin // Exclusive Store - mst_idx_t new_mask = id_mask_q[am_ex_addr_i] & am_ex_id_i; - id_mask_d[am_ex_addr_i] = new_mask; - am_ex_okay_o = !(new_mask == 0); + static mst_idx_t new_mask = id_mask_q[am_ex_addr_i] & am_ex_id_i; + id_mask_d[am_ex_addr_i] = new_mask; + am_ex_okay_o = !(new_mask == 0); end else if (am_ex_load_i) begin // Exclusive Load id_mask_d[am_ex_addr_i] = id_mask_q[am_ex_addr_i] | am_ex_id_i; diff --git a/src/ccu/ace_ccu_master_path.sv b/src/ccu/ace_ccu_master_path.sv index 74fae62..2e4d50d 100644 --- a/src/ccu/ace_ccu_master_path.sv +++ b/src/ccu/ace_ccu_master_path.sv @@ -382,7 +382,8 @@ module ace_ccu_master_path import ace_pkg::*; .snoop_req_t (snoop_req_t), .snoop_resp_t (snoop_resp_t), .domain_mask_t (domain_mask_t), - .domain_set_t (domain_set_t) + .domain_set_t (domain_set_t), + .mst_idx_t (mst_idx_t) ) i_snoop_path ( .clk_i (clk_i), .rst_ni (rst_ni), diff --git a/src/ccu/ace_ccu_snoop_interconnect.sv b/src/ccu/ace_ccu_snoop_interconnect.sv index 5826adf..c4e5742 100644 --- a/src/ccu/ace_ccu_snoop_interconnect.sv +++ b/src/ccu/ace_ccu_snoop_interconnect.sv @@ -10,7 +10,6 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( parameter int unsigned CmAddrBase = 0, parameter int unsigned AmAddrWidth = 0, parameter int unsigned AmAddrBase = 0, - parameter type NumMasters = 0, parameter type ac_chan_t = logic, parameter type cr_chan_t = logic, parameter type cd_chan_t = logic, @@ -19,8 +18,7 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( parameter type mst_idx_t = logic, localparam type oup_sel_t = logic [NumOup-1:0], - localparam type cm_addr_t = logic [CmAddrWidth-1:0], - localparam type am_addr_t = logic [AmAddrWidth-1:0] + localparam type cm_addr_t = logic [CmAddrWidth-1:0] ) ( input logic clk_i, @@ -40,9 +38,7 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( output logic cm_valid_o, output logic cm_ready_o, output cm_addr_t cm_addr_o, - input logic cm_stall_i, - - input logic [NumInp/2-1:0] + input logic cm_stall_i ); typedef logic [$clog2(NumInp)-1:0] inp_idx_t; @@ -64,7 +60,6 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( oup_sel_t [NumInp-1:0] inp_sel; mst_idx_t [NumInp-1:0] inp_idx; - logic [NumOup-1:0] oup_ac_valids, oup_ac_readies; ac_chan_t [NumOup-1:0] oup_ac_chans; logic [NumOup-1:0] oup_cr_valids, oup_cr_readies; @@ -77,7 +72,6 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( logic oup_excl_load, oup_excl_store; ac_chan_t ac_chan; mst_idx_t ac_mst_idx; - am_addr_t am_ex_addr; logic req_ctrl_valid, req_ctrl_ready; logic resp_ctrl_valid, resp_ctrl_ready; @@ -140,9 +134,16 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( assign inp_excl_store[i] = excl_store_i[i]; end if (BufferInpResp) begin : gen_buffer_resp + + typedef struct packed { + snoop_resp_t cr_resp; + logic excl_okay; + } cr_bundle_t; + cr_bundle_t cr_in, cr_out; + stream_fifo_optimal_wrap #( .Depth (2), - .type_t (cr_chan_t) + .type_t (cr_bundle_t) ) i_cr_fifo ( .clk_i, .rst_ni, @@ -151,10 +152,10 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( .usage_o (), .valid_i (inp_cr_valids [i]), .ready_o (inp_cr_readies[i]), - .data_i ({inp_cr_chans [i], cr_ctrl.excl_okay}), + .data_i (cr_in), .valid_o (inp_resp_o[i].cr_valid), .ready_i (inp_req_i [i].cr_ready), - .data_o ({inp_resp_o[i].cr_resp, excl_resp_o[i]}) + .data_o (cr_out) ); stream_fifo_optimal_wrap #( .Depth (4), @@ -172,6 +173,10 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( .ready_i (inp_req_i [i].cd_ready), .data_o (inp_resp_o[i].cd) ); + assign cr_in.cr_resp = inp_cr_chans[i]; + assign cr_in.excl_okay = cr_ctrl.excl_okay; + assign inp_resp_o[i].cr_resp = cr_out.cr_resp; + assign excl_resp_o[i] = cr_out.excl_okay; end else begin : gen_no_buffer_resp assign inp_cr_readies[i] = inp_req_i[i].cr_ready; assign inp_resp_o[i].cr_valid = inp_cr_valids[i]; @@ -250,52 +255,36 @@ module ace_ccu_snoop_interconnect import ace_pkg::*; #( end ace_ccu_snoop_req #( - .NumInp (NumInp), - .NumOup (NumOup), - .ac_chan_t (ac_chan_t), - .ctrl_t (ctrl_t) + .NumInp (NumInp), + .NumOup (NumOup), + .AmAddrWidth (AmAddrWidth), + .AmAddrBase (AmAddrBase), + .ac_chan_t (ac_chan_t), + .ctrl_t (ctrl_t), + .mst_idx_t (mst_idx_t) ) i_snoop_req ( .clk_i, .rst_ni, .ac_valids_i (inp_ac_valids), .ac_readies_o (inp_ac_readies), .ac_chans_i (inp_ac_chans), + .ac_mst_idxs_i (inp_idx), .ac_sel_i (inp_sel), .excl_load_i (inp_excl_load), .excl_store_i (inp_excl_store), .ac_valid_o (ac_valid), .ac_ready_i (ac_ready), .ac_chan_o (ac_chan), - .ac_mst_idx_o (ac_mst_idx), - .excl_load_o (oup_excl_load), - .excl_store_o (oup_excl_store), .ctrl_valid_o (req_ctrl_valid), .ctrl_ready_i (req_ctrl_ready), .ctrl_o (req_ctrl) ); - ace_ccu_atomic_manager #( - .AmIdxWidth (AmAddrWidth), - .mst_idx_t (mst_idx_t) - ) i_atomic_manager ( - .clk_i, - .rst_ni, - .am_ex_load_i (am_ex_load), - .am_ex_store_i (am_ex_store), - .am_ex_addr_i (am_ex_addr), - .am_ex_id_i (ac_mst_idx), - .am_ex_okay_o (excl_okay) - ); - - assign am_ex_addr = ac_chan.addr[AmAddrBase+:AmAddrWidth]; - assign am_ex_load = ac_valid && oup_excl_load; - assign am_ex_store = ac_valid && oup_excl_store; assign cm_valid_o = ac_valid; assign cm_ready_o = oup_ac_ready; assign cm_addr_o = ac_chan.addr[CmAddrBase+:CmAddrWidth]; assign ac_ready = oup_ac_ready && !cm_stall_i; assign oup_ac_valid = ac_valid && !cm_stall_i; - assign req_ctrl.excl_okay = excl_okay; stream_fifo_optimal_wrap #( .Depth (2), diff --git a/src/ccu/ace_ccu_snoop_path.sv b/src/ccu/ace_ccu_snoop_path.sv index 98ccba7..8bf754b 100644 --- a/src/ccu/ace_ccu_snoop_path.sv +++ b/src/ccu/ace_ccu_snoop_path.sv @@ -96,6 +96,7 @@ module ace_ccu_snoop_path import ace_pkg::*; import ccu_pkg::*; #( .mst_snoop_resp_t (snoop_resp_t), .domain_set_t (domain_set_t), .domain_mask_t (domain_mask_t), + .mst_idx_t (mst_idx_t), .AXLEN (WB_AXLEN), .AXSIZE (WB_AXSIZE), .FIFO_DEPTH (2) @@ -149,6 +150,7 @@ module ace_ccu_snoop_path import ace_pkg::*; import ccu_pkg::*; #( .mst_snoop_resp_t (snoop_resp_t), .domain_set_t (domain_set_t), .domain_mask_t (domain_mask_t), + .mst_idx_t (mst_idx_t), .AXLEN (WB_AXLEN), .AXSIZE (WB_AXSIZE), .FIFO_DEPTH (2) diff --git a/src/ccu/ace_ccu_snoop_req.sv b/src/ccu/ace_ccu_snoop_req.sv index 383fb21..af9d314 100644 --- a/src/ccu/ace_ccu_snoop_req.sv +++ b/src/ccu/ace_ccu_snoop_req.sv @@ -1,9 +1,13 @@ module ace_ccu_snoop_req #( - parameter int unsigned NumInp = 0, - parameter int unsigned NumOup = 0, - parameter type ac_chan_t = logic, - parameter type ctrl_t = logic, - parameter type mst_idx_t = logic + parameter int unsigned NumInp = 0, + parameter int unsigned NumOup = 0, + parameter int unsigned AmAddrWidth = 0, + parameter int unsigned AmAddrBase = 0, + parameter type ac_chan_t = logic, + parameter type ctrl_t = logic, + parameter type mst_idx_t = logic, + + localparam type am_addr_t = logic [AmAddrWidth-1:0] ) ( input logic clk_i, @@ -18,9 +22,6 @@ module ace_ccu_snoop_req #( output logic ac_valid_o, input logic ac_ready_i, output ac_chan_t ac_chan_o, - output mst_idx_t ac_mst_idx_o, - output logic excl_load_o, - output logic excl_store_o, output logic ctrl_valid_o, input logic ctrl_ready_i, output ctrl_t ctrl_o @@ -31,6 +32,10 @@ logic [NumOup-1:0] ac_sel; logic ac_valid, ac_ready; +am_addr_t am_ex_addr; +mst_idx_t am_ex_mst_idx; +logic excl_load, excl_store; + rr_arb_tree #( .NumIn (NumInp), .DataType (ac_chan_t), @@ -51,10 +56,11 @@ rr_arb_tree #( .idx_o (ac_idx) ); -assign ac_sel = ac_sel_i [ac_idx]; -assign ac_mst_idx_o = ac_mst_idxs_i[ac_idx]; -assign excl_load_o = excl_load_i [ac_idx]; -assign excl_store_o = excl_store_i [ac_idx]; +assign ac_sel = ac_sel_i [ac_idx]; +assign am_ex_mst_idx = ac_mst_idxs_i [ac_idx]; +assign excl_load = excl_load_i [ac_idx]; +assign excl_store = excl_store_i [ac_idx]; +assign am_ex_addr = ac_chan_o.addr[AmAddrBase+:AmAddrWidth]; stream_fork #( .N_OUP (2) @@ -67,6 +73,19 @@ stream_fork #( .ready_i ({ac_ready_i, ctrl_ready_i}) ); -assign ctrl_o = '{sel: ac_sel, idx: ac_idx}; +ace_ccu_atomic_manager #( + .AmIdxWidth (AmAddrWidth), + .mst_idx_t (mst_idx_t) +) i_atomic_manager ( + .clk_i, + .rst_ni, + .am_ex_load_i (excl_load), + .am_ex_store_i (excl_store), + .am_ex_addr_i (am_ex_addr), + .am_ex_id_i (am_ex_mst_idx), + .am_ex_okay_o (excl_okay) +); + +assign ctrl_o = '{sel: ac_sel, idx: ac_idx, excl_okay: excl_okay}; endmodule diff --git a/src/ccu/ace_ccu_top.sv b/src/ccu/ace_ccu_top.sv index 1da1278..0ffa48f 100644 --- a/src/ccu/ace_ccu_top.sv +++ b/src/ccu/ace_ccu_top.sv @@ -13,7 +13,8 @@ module ace_ccu_top import ace_pkg::*; parameter int unsigned DcacheLineWidth = 0, parameter int unsigned CmAddrBase = $clog2(DcacheLineWidth >> 3), parameter int unsigned CmAddrWidth = 8, - parameter int unsigned AmAddrWidth = 8, + parameter int unsigned AmAddrWidth = CmAddrWidth, + parameter int unsigned AmAddrBase = CmAddrBase, parameter type slv_ar_chan_t = logic, parameter type slv_aw_chan_t = logic, parameter type slv_b_chan_t = logic, @@ -78,7 +79,6 @@ module ace_ccu_top import ace_pkg::*; logic [2*NoGroups-1:0] excl_store; logic [2*NoGroups-1:0] excl_resp; mst_idx_t am_ex_id; - am_idx_t am_ex_addr; ace_ccu_master_path #( .AxiAddrWidth (AxiAddrWidth), @@ -110,7 +110,7 @@ module ace_ccu_top import ace_pkg::*; .snoop_resp_t (snoop_resp_t), .domain_set_t (domain_set_t), .domain_mask_t (domain_mask_t), - .mst_idx_t (domain_mask_t) + .mst_idx_t (mst_idx_t) ) i_master_path ( .clk_i (clk_i), .rst_ni (rst_ni), @@ -139,12 +139,14 @@ module ace_ccu_top import ace_pkg::*; .BufferOupResp(1), .CmAddrBase (CmAddrBase), .CmAddrWidth (CmAddrWidth), + .AmAddrWidth (AmAddrWidth), + .AmAddrBase (AmAddrBase), .ac_chan_t (snoop_ac_t), .cr_chan_t (snoop_cr_t), .cd_chan_t (snoop_cd_t), .snoop_req_t (snoop_req_t), .snoop_resp_t (snoop_resp_t), - .mst_idx_t (domain_mask_t) + .mst_idx_t (mst_idx_t) ) i_snoop_interconnect ( .clk_i (clk_i), .rst_ni (rst_ni), @@ -154,11 +156,13 @@ module ace_ccu_top import ace_pkg::*; .inp_resp_o (snoop_resps), .oup_req_o (snoop_req_o), .oup_resp_i (snoop_resp_i), + .excl_load_i (excl_load), + .excl_store_i (excl_store), + .excl_resp_o (excl_resp), .cm_valid_o (cm_snoop_valid), .cm_ready_o (cm_snoop_ready), .cm_addr_o (cm_snoop_addr), - .cm_stall_i (cm_snoop_stall), - .am_ex_id_o (am_ex_id) + .cm_stall_i (cm_snoop_stall) ); ace_ccu_conflict_manager #( diff --git a/src/ccu/ccu_ctrl_r_snoop.sv b/src/ccu/ccu_ctrl_r_snoop.sv index 278e17f..031e1a5 100644 --- a/src/ccu/ccu_ctrl_r_snoop.sv +++ b/src/ccu/ccu_ctrl_r_snoop.sv @@ -262,7 +262,7 @@ always_comb begin WRITE_CD: begin mst_req_o.w_valid = cd_fork_valid[MEM_W_IDX] && !aw_valid_q; slv_resp_o.r.data = snoop_resp_i.cd.data; - slv_resp_o.r.resp = {rresp_q[3:2], 1'b0, excl_store_resp_i}; + slv_resp_o.r.resp = {rresp_q[3:2], 1'b0, excl_resp_i}; slv_resp_o.r.last = r_last; slv_resp_o.r_valid = cd_fork_valid[MST_R_IDX] && !r_last_q; arlen_counter_en = r_handshake; From 84b59e3ca41d8c767ffe4b77d3f25da20091b27f Mon Sep 17 00:00:00 2001 From: "Aleksi Korsman aleksi.korsman@aalto.fi" Date: Mon, 25 Nov 2024 17:46:08 +0100 Subject: [PATCH 7/7] Use correct rresp signals --- src/ccu/ccu_ctrl_r_snoop.sv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ccu/ccu_ctrl_r_snoop.sv b/src/ccu/ccu_ctrl_r_snoop.sv index 031e1a5..29856c8 100644 --- a/src/ccu/ccu_ctrl_r_snoop.sv +++ b/src/ccu/ccu_ctrl_r_snoop.sv @@ -262,7 +262,7 @@ always_comb begin WRITE_CD: begin mst_req_o.w_valid = cd_fork_valid[MEM_W_IDX] && !aw_valid_q; slv_resp_o.r.data = snoop_resp_i.cd.data; - slv_resp_o.r.resp = {rresp_q[3:2], 1'b0, excl_resp_i}; + slv_resp_o.r.resp = rresp_q; slv_resp_o.r.last = r_last; slv_resp_o.r_valid = cd_fork_valid[MST_R_IDX] && !r_last_q; arlen_counter_en = r_handshake; @@ -301,6 +301,7 @@ always_comb begin ar_valid_d = 1'b0; end slv_resp_o.r = mst_resp_i.r; + slv_resp_o.r.resp = rresp_q; slv_resp_o.r_valid = mst_resp_i.r_valid; mst_req_o.r_ready = slv_req_i.r_ready; if (r_handshake && slv_resp_o.r.last) begin