1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #ifndef _ROC_NIX_PRIV_H_ 6 #define _ROC_NIX_PRIV_H_ 7 8 /* Constants */ 9 #define NIX_CQ_ENTRY_SZ 128 10 #define NIX_CQ_ENTRY64_SZ 512 11 #define NIX_CQ_ALIGN ((uint16_t)512) 12 #define NIX_MAX_SQB ((uint16_t)512) 13 #define NIX_DEF_SQB ((uint16_t)16) 14 #define NIX_MIN_SQB ((uint16_t)8) 15 #define NIX_SQB_PREFETCH ((uint16_t)1) 16 17 /* Apply BP/DROP when CQ is 95% full */ 18 #define NIX_CQ_THRESH_LEVEL (5 * 256 / 100) 19 #define NIX_CQ_SEC_THRESH_LEVEL (25 * 256 / 100) 20 /* Apply LBP at 75% of actual BP */ 21 #define NIX_CQ_LPB_THRESH_FRAC (75 * 16 / 100) 22 #define NIX_CQ_FULL_ERRATA_SKID (1024ull * 256) 23 #define NIX_RQ_AURA_BP_THRESH(percent, limit, shift) ((((limit) * (percent)) / 100) >> (shift)) 24 25 /* IRQ triggered when NIX_LF_CINTX_CNT[QCOUNT] crosses this value */ 26 #define CQ_CQE_THRESH_DEFAULT 0x1ULL 27 #define CQ_TIMER_THRESH_DEFAULT 0xAULL /* ~1usec i.e (0xA * 100nsec) */ 28 #define CQ_TIMER_THRESH_MAX 255 29 30 struct nix_qint { 31 struct nix *nix; 32 uint8_t qintx; 33 }; 34 35 /* Traffic Manager */ 36 #define NIX_TM_MAX_HW_TXSCHQ 1024 37 #define NIX_TM_HW_ID_INVALID UINT32_MAX 38 #define NIX_TM_CHAN_INVALID UINT16_MAX 39 40 /* TM flags */ 41 #define NIX_TM_HIERARCHY_ENA BIT_ULL(0) 42 #define NIX_TM_TL1_NO_SP BIT_ULL(1) 43 #define NIX_TM_TL1_ACCESS BIT_ULL(2) 44 #define NIX_TM_MARK_VLAN_DEI_EN BIT_ULL(3) 45 #define NIX_TM_MARK_IP_DSCP_EN BIT_ULL(4) 46 #define NIX_TM_MARK_IP_ECN_EN BIT_ULL(5) 47 48 #define NIX_TM_MARK_EN_MASK \ 49 (NIX_TM_MARK_IP_DSCP_EN | NIX_TM_MARK_IP_ECN_EN | \ 50 NIX_TM_MARK_VLAN_DEI_EN) 51 52 #define NIX_TM_MARK_VLAN_DEI_SHIFT 0 /* Leave 16b for VLAN for FP logic */ 53 #define NIX_TM_MARK_IPV4_DSCP_SHIFT 16 54 #define NIX_TM_MARK_IPV6_DSCP_SHIFT 24 55 #define NIX_TM_MARK_IPV4_ECN_SHIFT 32 56 #define NIX_TM_MARK_IPV6_ECN_SHIFT 40 57 58 struct nix_tm_tb { 59 /** Token bucket rate (bytes per second) */ 60 uint64_t rate; 61 62 /** Token bucket size (bytes), a.k.a. max burst size */ 63 uint64_t size; 64 }; 65 66 struct nix_tm_node { 67 TAILQ_ENTRY(nix_tm_node) node; 68 69 /* Input params */ 70 enum roc_nix_tm_tree tree; 71 uint32_t id; 72 uint32_t priority; 73 uint32_t weight; 74 uint16_t lvl; 75 uint16_t rel_chan; 76 uint32_t parent_id; 77 uint32_t shaper_profile_id; 78 void (*free_fn)(void *node); 79 80 /* Derived params */ 81 uint32_t hw_id; 82 uint16_t hw_lvl; 83 uint32_t rr_prio; 84 uint32_t rr_num; 85 uint32_t max_prio; 86 uint32_t parent_hw_id; 87 uint32_t flags : 16; 88 #define NIX_TM_NODE_HWRES BIT_ULL(0) 89 #define NIX_TM_NODE_ENABLED BIT_ULL(1) 90 /* Shaper algorithm for RED state @NIX_REDALG_E */ 91 uint32_t red_algo : 2; 92 uint32_t pkt_mode : 1; 93 uint32_t pkt_mode_set : 1; 94 uint32_t bp_capa : 1; 95 96 bool child_realloc; 97 struct nix_tm_node *parent; 98 99 /* Non-leaf node sp count */ 100 uint32_t n_sp_priorities; 101 102 /* Last stats */ 103 uint64_t last_pkts; 104 uint64_t last_bytes; 105 }; 106 107 struct nix_tm_shaper_profile { 108 TAILQ_ENTRY(nix_tm_shaper_profile) shaper; 109 struct nix_tm_tb commit; 110 struct nix_tm_tb peak; 111 int32_t pkt_len_adj; 112 int32_t pkt_mode_adj; 113 bool pkt_mode; 114 uint32_t id; 115 int8_t accuracy; 116 void (*free_fn)(void *profile); 117 118 uint32_t ref_cnt; 119 }; 120 121 TAILQ_HEAD(nix_tm_node_list, nix_tm_node); 122 TAILQ_HEAD(nix_tm_shaper_profile_list, nix_tm_shaper_profile); 123 124 struct nix { 125 uint16_t reta[ROC_NIX_RSS_GRPS][ROC_NIX_RSS_RETA_MAX]; 126 enum roc_nix_rss_reta_sz reta_sz; 127 struct plt_pci_device *pci_dev; 128 uint16_t bpid[NIX_MAX_CHAN]; 129 struct nix_qint *qints_mem; 130 struct nix_qint *cints_mem; 131 uint8_t configured_qints; 132 uint8_t configured_cints; 133 uint8_t exact_match_ena; 134 struct roc_nix_rq **rqs; 135 struct roc_nix_sq **sqs; 136 uint16_t vwqe_interval; 137 uint16_t tx_chan_base; 138 uint16_t rx_chan_base; 139 uint16_t nb_rx_queues; 140 uint16_t nb_tx_queues; 141 uint8_t lso_tsov6_idx; 142 uint8_t lso_tsov4_idx; 143 uint8_t lso_udp_tun_idx[ROC_NIX_LSO_TUN_MAX]; 144 uint8_t lso_tun_idx[ROC_NIX_LSO_TUN_MAX]; 145 uint8_t lf_rx_stats; 146 uint8_t lf_tx_stats; 147 uint8_t rx_chan_cnt; 148 uint8_t rss_alg_idx; 149 uint8_t tx_chan_cnt; 150 uintptr_t lmt_base; 151 uint8_t cgx_links; 152 uint8_t lbk_links; 153 uint8_t sdp_links; 154 uint8_t tx_link; 155 uint16_t sqb_size; 156 uint32_t dmac_flt_idx; 157 /* Without FCS, with L2 overhead */ 158 uint16_t mtu; 159 uint16_t chan_cnt; 160 uint16_t msixoff; 161 uint8_t rx_pause; 162 uint8_t tx_pause; 163 uint8_t pfc_rx_pause; 164 uint8_t pfc_tx_pause; 165 uint16_t cev; 166 uint64_t rx_cfg; 167 struct dev dev; 168 uint16_t cints; 169 uint16_t qints; 170 uintptr_t base; 171 bool sdp_link; 172 bool lbk_link; 173 bool esw_link; 174 bool ptp_en; 175 bool is_nix1; 176 177 /* Traffic manager info */ 178 179 /* Contiguous resources per lvl */ 180 struct plt_bitmap *schq_contig_bmp[NIX_TXSCH_LVL_CNT]; 181 /* Dis-contiguous resources per lvl */ 182 struct plt_bitmap *schq_bmp[NIX_TXSCH_LVL_CNT]; 183 void *schq_bmp_mem; 184 185 struct nix_tm_shaper_profile_list shaper_profile_list; 186 struct nix_tm_node_list trees[ROC_NIX_TM_TREE_MAX]; 187 enum roc_nix_tm_tree tm_tree; 188 uint64_t tm_rate_min; 189 uint16_t tm_root_lvl; 190 uint16_t tm_flags; 191 uint16_t tm_link_cfg_lvl; 192 uint8_t tm_aggr_lvl_rr_prio; 193 uint16_t contig_rsvd[NIX_TXSCH_LVL_CNT]; 194 uint16_t discontig_rsvd[NIX_TXSCH_LVL_CNT]; 195 uint64_t tm_markfmt_en; 196 uint8_t tm_markfmt_null; 197 uint8_t tm_markfmt[ROC_NIX_TM_MARK_MAX][ROC_NIX_TM_MARK_COLOR_MAX]; 198 199 /* Ipsec info */ 200 uint16_t cpt_msixoff[MAX_RVU_BLKLF_CNT]; 201 bool inl_inb_ena; 202 bool inl_outb_ena; 203 void *inb_sa_base; 204 size_t inb_sa_sz; 205 uint32_t inb_spi_mask; 206 void *outb_sa_base; 207 size_t outb_sa_sz; 208 uint16_t outb_err_sso_pffunc; 209 struct roc_cpt_lf *cpt_lf_base; 210 uint16_t nb_cpt_lf; 211 uint16_t outb_se_ring_cnt; 212 uint16_t outb_se_ring_base; 213 uint16_t cpt_lbpid; 214 uint16_t cpt_nixbpid; 215 uint64_t cpt_eng_caps; 216 bool need_meta_aura; 217 /* Mode provided by driver */ 218 bool inb_inl_dev; 219 220 } __plt_cache_aligned; 221 222 enum nix_err_status { 223 NIX_ERR_PARAM = -2048, 224 NIX_ERR_NO_MEM, 225 NIX_ERR_INVALID_RANGE, 226 NIX_ERR_INTERNAL, 227 NIX_ERR_OP_NOTSUP, 228 NIX_ERR_HW_NOTSUP, 229 NIX_ERR_QUEUE_INVALID_RANGE, 230 NIX_ERR_AQ_READ_FAILED, 231 NIX_ERR_AQ_WRITE_FAILED, 232 NIX_ERR_TM_LEAF_NODE_GET, 233 NIX_ERR_TM_INVALID_LVL, 234 NIX_ERR_TM_INVALID_PRIO, 235 NIX_ERR_TM_INVALID_PARENT, 236 NIX_ERR_TM_NODE_EXISTS, 237 NIX_ERR_TM_INVALID_NODE, 238 NIX_ERR_TM_INVALID_SHAPER_PROFILE, 239 NIX_ERR_TM_PKT_MODE_MISMATCH, 240 NIX_ERR_TM_WEIGHT_EXCEED, 241 NIX_ERR_TM_CHILD_EXISTS, 242 NIX_ERR_TM_INVALID_PEAK_SZ, 243 NIX_ERR_TM_INVALID_PEAK_RATE, 244 NIX_ERR_TM_INVALID_COMMIT_SZ, 245 NIX_ERR_TM_INVALID_COMMIT_RATE, 246 NIX_ERR_TM_SHAPER_PROFILE_IN_USE, 247 NIX_ERR_TM_SHAPER_PROFILE_EXISTS, 248 NIX_ERR_TM_SHAPER_PKT_LEN_ADJUST, 249 NIX_ERR_TM_INVALID_TREE, 250 NIX_ERR_TM_PARENT_PRIO_UPDATE, 251 NIX_ERR_TM_PRIO_EXCEEDED, 252 NIX_ERR_TM_PRIO_ORDER, 253 NIX_ERR_TM_MULTIPLE_RR_GROUPS, 254 NIX_ERR_TM_SQ_UPDATE_FAIL, 255 NIX_ERR_NDC_SYNC, 256 }; 257 258 enum nix_q_size { 259 nix_q_size_16, /* 16 entries */ 260 nix_q_size_64, /* 64 entries */ 261 nix_q_size_256, 262 nix_q_size_1K, 263 nix_q_size_4K, 264 nix_q_size_16K, 265 nix_q_size_64K, 266 nix_q_size_256K, 267 nix_q_size_1M, /* Million entries */ 268 nix_q_size_max 269 }; 270 271 static inline struct nix * 272 roc_nix_to_nix_priv(struct roc_nix *roc_nix) 273 { 274 return (struct nix *)&roc_nix->reserved[0]; 275 } 276 277 static inline struct roc_nix * 278 nix_priv_to_roc_nix(struct nix *nix) 279 { 280 return (struct roc_nix *)((char *)nix - 281 offsetof(struct roc_nix, reserved)); 282 } 283 284 /* IRQ */ 285 int nix_register_irqs(struct nix *nix); 286 void nix_unregister_irqs(struct nix *nix); 287 288 /* TM */ 289 #define NIX_TM_TREE_MASK_ALL \ 290 (BIT(ROC_NIX_TM_DEFAULT) | BIT(ROC_NIX_TM_RLIMIT) | \ 291 BIT(ROC_NIX_TM_USER)) 292 293 /* NIX_MAX_HW_FRS == 294 * NIX_TM_DFLT_RR_WT * NIX_TM_RR_QUANTUM_MAX / ROC_NIX_TM_MAX_SCHED_WT 295 */ 296 #define NIX_TM_DFLT_RR_WT 71 297 298 /* Default TL1 priority and Quantum from AF */ 299 #define NIX_TM_TL1_DFLT_RR_QTM ((1 << 24) - 1) 300 301 struct nix_tm_shaper_data { 302 uint64_t burst_exponent; 303 uint64_t burst_mantissa; 304 uint64_t div_exp; 305 uint64_t exponent; 306 uint64_t mantissa; 307 uint64_t burst; 308 uint64_t rate; 309 }; 310 311 static inline uint64_t 312 nix_tm_weight_to_rr_quantum(uint64_t weight) 313 { 314 uint64_t max = NIX_CN9K_TM_RR_QUANTUM_MAX; 315 316 /* From CN10K onwards, we only configure RR weight */ 317 if (!roc_model_is_cn9k()) 318 return weight; 319 320 weight &= (uint64_t)max; 321 return (weight * max) / ROC_NIX_CN9K_TM_RR_WEIGHT_MAX; 322 } 323 324 static inline bool 325 nix_tm_have_tl1_access(struct nix *nix) 326 { 327 return !!(nix->tm_flags & NIX_TM_TL1_ACCESS); 328 } 329 330 static inline bool 331 nix_tm_is_leaf(struct nix *nix, int lvl) 332 { 333 if (nix_tm_have_tl1_access(nix)) 334 return (lvl == ROC_TM_LVL_QUEUE); 335 return (lvl == ROC_TM_LVL_SCH4); 336 } 337 338 static inline struct nix_tm_node_list * 339 nix_tm_node_list(struct nix *nix, enum roc_nix_tm_tree tree) 340 { 341 return &nix->trees[tree]; 342 } 343 344 static inline const char * 345 nix_tm_hwlvl2str(uint32_t hw_lvl) 346 { 347 switch (hw_lvl) { 348 case NIX_TXSCH_LVL_MDQ: 349 return "SMQ/MDQ"; 350 case NIX_TXSCH_LVL_TL4: 351 return "TL4"; 352 case NIX_TXSCH_LVL_TL3: 353 return "TL3"; 354 case NIX_TXSCH_LVL_TL2: 355 return "TL2"; 356 case NIX_TXSCH_LVL_TL1: 357 return "TL1"; 358 default: 359 break; 360 } 361 362 return "???"; 363 } 364 365 static inline const char * 366 nix_tm_tree2str(enum roc_nix_tm_tree tree) 367 { 368 if (tree == ROC_NIX_TM_DEFAULT) 369 return "Default Tree"; 370 else if (tree == ROC_NIX_TM_RLIMIT) 371 return "Rate Limit Tree"; 372 else if (tree == ROC_NIX_TM_PFC) 373 return "PFC Tree"; 374 else if (tree == ROC_NIX_TM_USER) 375 return "User Tree"; 376 return "???"; 377 } 378 379 /* 380 * TM priv ops. 381 */ 382 383 int nix_tm_conf_init(struct roc_nix *roc_nix); 384 void nix_tm_conf_fini(struct roc_nix *roc_nix); 385 int nix_tm_leaf_data_get(struct nix *nix, uint16_t sq, uint32_t *rr_quantum, 386 uint16_t *smq); 387 int nix_tm_sq_flush_pre(struct roc_nix_sq *sq); 388 int nix_tm_sq_flush_post(struct roc_nix_sq *sq); 389 int nix_tm_smq_xoff(struct nix *nix, struct nix_tm_node *node, bool enable); 390 int nix_tm_prepare_default_tree(struct roc_nix *roc_nix); 391 int nix_tm_node_add(struct roc_nix *roc_nix, struct nix_tm_node *node); 392 int nix_tm_node_delete(struct roc_nix *roc_nix, uint32_t node_id, 393 enum roc_nix_tm_tree tree, bool free); 394 int nix_tm_free_node_resource(struct nix *nix, struct nix_tm_node *node); 395 int nix_tm_free_resources(struct roc_nix *roc_nix, uint32_t tree_mask, 396 bool hw_only); 397 int nix_tm_clear_path_xoff(struct nix *nix, struct nix_tm_node *node); 398 void nix_tm_clear_shaper_profiles(struct nix *nix); 399 int nix_tm_alloc_txschq(struct nix *nix, enum roc_nix_tm_tree tree); 400 int nix_tm_assign_resources(struct nix *nix, enum roc_nix_tm_tree tree); 401 int nix_tm_release_resources(struct nix *nix, uint8_t hw_lvl, bool contig, 402 bool above_thresh); 403 void nix_tm_copy_rsp_to_nix(struct nix *nix, struct nix_txsch_alloc_rsp *rsp); 404 405 int nix_tm_txsch_reg_config(struct nix *nix, enum roc_nix_tm_tree tree); 406 int nix_tm_update_parent_info(struct nix *nix, enum roc_nix_tm_tree tree); 407 int nix_tm_sq_sched_conf(struct nix *nix, struct nix_tm_node *node, 408 bool rr_quantum_only); 409 410 int nix_rq_cn9k_cfg(struct dev *dev, struct roc_nix_rq *rq, uint16_t qints, 411 bool cfg, bool ena); 412 int nix_rq_cn10k_cfg(struct dev *dev, struct roc_nix_rq *rq, uint16_t qints, bool cfg, bool ena); 413 int nix_rq_cfg(struct dev *dev, struct roc_nix_rq *rq, uint16_t qints, bool cfg, 414 bool ena); 415 int nix_rq_ena_dis(struct dev *dev, struct roc_nix_rq *rq, bool enable); 416 int nix_tm_bp_config_get(struct roc_nix *roc_nix, bool *is_enabled); 417 int nix_tm_bp_config_set(struct roc_nix *roc_nix, uint16_t sq, uint16_t tc, 418 bool enable); 419 void nix_rq_vwqe_flush(struct roc_nix_rq *rq, uint16_t vwqe_interval); 420 int nix_tm_mark_init(struct nix *nix); 421 void nix_tm_sq_free_sqe_buffer(uint64_t *sqe, int head_off, int end_off, int instr_sz); 422 int roc_nix_tm_sq_free_pending_sqe(struct nix *nix, int q); 423 424 /* 425 * TM priv utils. 426 */ 427 uint16_t nix_tm_lvl2nix(struct nix *nix, uint32_t lvl); 428 uint16_t nix_tm_lvl2nix_tl1_root(uint32_t lvl); 429 uint16_t nix_tm_lvl2nix_tl2_root(uint32_t lvl); 430 uint16_t nix_tm_resource_avail(struct nix *nix, uint8_t hw_lvl, bool contig); 431 int nix_tm_validate_prio(struct nix *nix, uint32_t lvl, uint32_t parent_id, 432 uint32_t priority, enum roc_nix_tm_tree tree); 433 struct nix_tm_node *nix_tm_node_search(struct nix *nix, uint32_t node_id, 434 enum roc_nix_tm_tree tree); 435 struct nix_tm_shaper_profile *nix_tm_shaper_profile_search(struct nix *nix, 436 uint32_t id); 437 uint8_t nix_tm_sw_xoff_prep(struct nix_tm_node *node, bool enable, 438 volatile uint64_t *reg, volatile uint64_t *regval); 439 uint32_t nix_tm_check_rr(struct nix *nix, uint32_t parent_id, 440 enum roc_nix_tm_tree tree, uint32_t *rr_prio, 441 uint32_t *max_prio); 442 uint64_t nix_tm_shaper_profile_rate_min(struct nix *nix); 443 uint64_t nix_tm_shaper_rate_conv(uint64_t value, uint64_t *exponent_p, 444 uint64_t *mantissa_p, uint64_t *div_exp_p, 445 int8_t accuracy); 446 uint64_t nix_tm_shaper_burst_conv(uint64_t value, uint64_t *exponent_p, 447 uint64_t *mantissa_p); 448 bool nix_tm_child_res_valid(struct nix_tm_node_list *list, 449 struct nix_tm_node *parent); 450 uint16_t nix_tm_resource_estimate(struct nix *nix, uint16_t *schq_contig, 451 uint16_t *schq, enum roc_nix_tm_tree tree); 452 uint8_t nix_tm_tl1_default_prep(struct nix *nix, uint32_t schq, 453 volatile uint64_t *reg, 454 volatile uint64_t *regval); 455 uint8_t nix_tm_topology_reg_prep(struct nix *nix, struct nix_tm_node *node, 456 volatile uint64_t *reg, 457 volatile uint64_t *regval, 458 volatile uint64_t *regval_mask); 459 uint8_t nix_tm_sched_reg_prep(struct nix *nix, struct nix_tm_node *node, 460 volatile uint64_t *reg, 461 volatile uint64_t *regval); 462 uint8_t nix_tm_shaper_reg_prep(struct nix_tm_node *node, 463 struct nix_tm_shaper_profile *profile, 464 volatile uint64_t *reg, 465 volatile uint64_t *regval); 466 struct nix_tm_node *nix_tm_node_alloc(void); 467 void nix_tm_node_free(struct nix_tm_node *node); 468 struct nix_tm_shaper_profile *nix_tm_shaper_profile_alloc(void); 469 void nix_tm_shaper_profile_free(struct nix_tm_shaper_profile *profile); 470 471 uint64_t nix_get_blkaddr(struct dev *dev); 472 void nix_cn10k_lf_rq_dump(__io struct nix_cn10k_rq_ctx_s *ctx, FILE *file); 473 void nix_lf_rq_dump(__io struct nix_cn20k_rq_ctx_s *ctx, FILE *file); 474 int nix_lf_gen_reg_dump(uintptr_t nix_lf_base, uint64_t *data); 475 int nix_lf_stat_reg_dump(uintptr_t nix_lf_base, uint64_t *data, uint8_t lf_tx_stats, 476 uint8_t lf_rx_stats); 477 int nix_lf_int_reg_dump(uintptr_t nix_lf_base, uint64_t *data, uint16_t qints, uint16_t cints); 478 int nix_q_ctx_get(struct dev *dev, uint8_t ctype, uint16_t qid, __io void **ctx_p); 479 uint8_t nix_tm_lbk_relchan_get(struct nix *nix); 480 int nix_vlan_tpid_set(struct mbox *mbox, uint16_t pcifunc, uint32_t type, uint16_t tpid); 481 482 /* 483 * Telemetry 484 */ 485 int nix_tel_node_add(struct roc_nix *roc_nix); 486 void nix_tel_node_del(struct roc_nix *roc_nix); 487 int nix_tel_node_add_rq(struct roc_nix_rq *rq); 488 int nix_tel_node_add_cq(struct roc_nix_cq *cq); 489 int nix_tel_node_add_sq(struct roc_nix_sq *sq); 490 491 /* 492 * RSS 493 */ 494 int nix_rss_reta_pffunc_set(struct roc_nix *roc_nix, uint8_t group, 495 uint16_t reta[ROC_NIX_RSS_RETA_MAX], uint16_t pf_func); 496 int nix_rss_flowkey_pffunc_set(struct roc_nix *roc_nix, uint8_t *alg_idx, uint32_t flowkey, 497 uint8_t group, int mcam_index, uint16_t pf_func); 498 499 #endif /* _ROC_NIX_PRIV_H_ */ 500