1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include <math.h> 6 7 #include "roc_api.h" 8 #include "roc_priv.h" 9 10 static inline uint32_t 11 nix_qsize_to_val(enum nix_q_size qsize) 12 { 13 return (16UL << (qsize * 2)); 14 } 15 16 static inline enum nix_q_size 17 nix_qsize_clampup(uint32_t val) 18 { 19 int i = nix_q_size_16; 20 21 for (; i < nix_q_size_max; i++) 22 if (val <= nix_qsize_to_val(i)) 23 break; 24 25 if (i >= nix_q_size_max) 26 i = nix_q_size_max - 1; 27 28 return i; 29 } 30 31 void 32 nix_rq_vwqe_flush(struct roc_nix_rq *rq, uint16_t vwqe_interval) 33 { 34 uint64_t wait_ns; 35 36 if (!roc_model_is_cn10k()) 37 return; 38 /* Due to HW errata writes to VWQE_FLUSH might hang, so instead 39 * wait for max vwqe timeout interval. 40 */ 41 if (rq->vwqe_ena) { 42 wait_ns = rq->vwqe_wait_tmo * (vwqe_interval + 1) * 100; 43 plt_delay_us((wait_ns / 1E3) + 1); 44 } 45 } 46 47 int 48 nix_rq_ena_dis(struct dev *dev, struct roc_nix_rq *rq, bool enable) 49 { 50 struct mbox *mbox = dev->mbox; 51 52 /* Pkts will be dropped silently if RQ is disabled */ 53 if (roc_model_is_cn9k()) { 54 struct nix_aq_enq_req *aq; 55 56 aq = mbox_alloc_msg_nix_aq_enq(mbox); 57 if (!aq) 58 return -ENOSPC; 59 60 aq->qidx = rq->qid; 61 aq->ctype = NIX_AQ_CTYPE_RQ; 62 aq->op = NIX_AQ_INSTOP_WRITE; 63 64 aq->rq.ena = enable; 65 aq->rq_mask.ena = ~(aq->rq_mask.ena); 66 } else { 67 struct nix_cn10k_aq_enq_req *aq; 68 69 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 70 if (!aq) 71 return -ENOSPC; 72 73 aq->qidx = rq->qid; 74 aq->ctype = NIX_AQ_CTYPE_RQ; 75 aq->op = NIX_AQ_INSTOP_WRITE; 76 77 aq->rq.ena = enable; 78 aq->rq_mask.ena = ~(aq->rq_mask.ena); 79 } 80 81 return mbox_process(mbox); 82 } 83 84 int 85 roc_nix_rq_ena_dis(struct roc_nix_rq *rq, bool enable) 86 { 87 struct nix *nix = roc_nix_to_nix_priv(rq->roc_nix); 88 int rc; 89 90 rc = nix_rq_ena_dis(&nix->dev, rq, enable); 91 nix_rq_vwqe_flush(rq, nix->vwqe_interval); 92 if (rc) 93 return rc; 94 95 /* Check for meta aura if RQ is enabled */ 96 if (enable && nix->need_meta_aura) 97 rc = roc_nix_inl_meta_aura_check(rq); 98 return rc; 99 } 100 101 int 102 roc_nix_rq_is_sso_enable(struct roc_nix *roc_nix, uint32_t qid) 103 { 104 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 105 struct dev *dev = &nix->dev; 106 struct mbox *mbox = dev->mbox; 107 bool sso_enable; 108 int rc; 109 110 if (roc_model_is_cn9k()) { 111 struct nix_aq_enq_rsp *rsp; 112 struct nix_aq_enq_req *aq; 113 114 aq = mbox_alloc_msg_nix_aq_enq(mbox); 115 if (!aq) 116 return -ENOSPC; 117 118 aq->qidx = qid; 119 aq->ctype = NIX_AQ_CTYPE_RQ; 120 aq->op = NIX_AQ_INSTOP_READ; 121 rc = mbox_process_msg(mbox, (void *)&rsp); 122 if (rc) 123 return rc; 124 125 sso_enable = rsp->rq.sso_ena; 126 } else { 127 struct nix_cn10k_aq_enq_rsp *rsp; 128 struct nix_cn10k_aq_enq_req *aq; 129 130 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 131 if (!aq) 132 return -ENOSPC; 133 134 aq->qidx = qid; 135 aq->ctype = NIX_AQ_CTYPE_RQ; 136 aq->op = NIX_AQ_INSTOP_READ; 137 138 rc = mbox_process_msg(mbox, (void *)&rsp); 139 if (rc) 140 return rc; 141 142 sso_enable = rsp->rq.sso_ena; 143 } 144 145 return sso_enable ? true : false; 146 } 147 148 static int 149 nix_rq_aura_buf_type_update(struct roc_nix_rq *rq, bool set) 150 { 151 struct roc_nix *roc_nix = rq->roc_nix; 152 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 153 bool inl_inb_ena = roc_nix_inl_inb_is_enabled(roc_nix); 154 uint64_t lpb_aura = 0, vwqe_aura = 0, spb_aura = 0; 155 struct mbox *mbox = nix->dev.mbox; 156 uint64_t aura_base; 157 int rc, count; 158 159 count = set ? 1 : -1; 160 /* For buf type set, use info from RQ context */ 161 if (set) { 162 lpb_aura = rq->aura_handle; 163 spb_aura = rq->spb_ena ? rq->spb_aura_handle : 0; 164 vwqe_aura = rq->vwqe_ena ? rq->vwqe_aura_handle : 0; 165 goto skip_ctx_read; 166 } 167 168 aura_base = roc_npa_aura_handle_to_base(rq->aura_handle); 169 if (roc_model_is_cn9k()) { 170 struct nix_aq_enq_rsp *rsp; 171 struct nix_aq_enq_req *aq; 172 173 aq = mbox_alloc_msg_nix_aq_enq(mbox); 174 if (!aq) 175 return -ENOSPC; 176 177 aq->qidx = rq->qid; 178 aq->ctype = NIX_AQ_CTYPE_RQ; 179 aq->op = NIX_AQ_INSTOP_READ; 180 rc = mbox_process_msg(mbox, (void *)&rsp); 181 if (rc) 182 return rc; 183 184 /* Get aura handle from aura */ 185 lpb_aura = roc_npa_aura_handle_gen(rsp->rq.lpb_aura, aura_base); 186 if (rsp->rq.spb_ena) 187 spb_aura = roc_npa_aura_handle_gen(rsp->rq.spb_aura, aura_base); 188 } else { 189 struct nix_cn10k_aq_enq_rsp *rsp; 190 struct nix_cn10k_aq_enq_req *aq; 191 192 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 193 if (!aq) 194 return -ENOSPC; 195 196 aq->qidx = rq->qid; 197 aq->ctype = NIX_AQ_CTYPE_RQ; 198 aq->op = NIX_AQ_INSTOP_READ; 199 200 rc = mbox_process_msg(mbox, (void *)&rsp); 201 if (rc) 202 return rc; 203 204 /* Get aura handle from aura */ 205 lpb_aura = roc_npa_aura_handle_gen(rsp->rq.lpb_aura, aura_base); 206 if (rsp->rq.spb_ena) 207 spb_aura = roc_npa_aura_handle_gen(rsp->rq.spb_aura, aura_base); 208 if (rsp->rq.vwqe_ena) 209 vwqe_aura = roc_npa_aura_handle_gen(rsp->rq.wqe_aura, aura_base); 210 } 211 212 skip_ctx_read: 213 /* Update attributes for LPB aura */ 214 if (inl_inb_ena) 215 roc_npa_buf_type_update(lpb_aura, ROC_NPA_BUF_TYPE_PACKET_IPSEC, count); 216 else 217 roc_npa_buf_type_update(lpb_aura, ROC_NPA_BUF_TYPE_PACKET, count); 218 219 /* Update attributes for SPB aura */ 220 if (spb_aura) { 221 if (inl_inb_ena) 222 roc_npa_buf_type_update(spb_aura, ROC_NPA_BUF_TYPE_PACKET_IPSEC, count); 223 else 224 roc_npa_buf_type_update(spb_aura, ROC_NPA_BUF_TYPE_PACKET, count); 225 } 226 227 /* Update attributes for VWQE aura */ 228 if (vwqe_aura) { 229 if (inl_inb_ena) 230 roc_npa_buf_type_update(vwqe_aura, ROC_NPA_BUF_TYPE_VWQE_IPSEC, count); 231 else 232 roc_npa_buf_type_update(vwqe_aura, ROC_NPA_BUF_TYPE_VWQE, count); 233 } 234 235 return 0; 236 } 237 238 int 239 nix_rq_cn9k_cfg(struct dev *dev, struct roc_nix_rq *rq, uint16_t qints, 240 bool cfg, bool ena) 241 { 242 struct mbox *mbox = dev->mbox; 243 struct nix_aq_enq_req *aq; 244 245 aq = mbox_alloc_msg_nix_aq_enq(mbox); 246 if (!aq) 247 return -ENOSPC; 248 249 aq->qidx = rq->qid; 250 aq->ctype = NIX_AQ_CTYPE_RQ; 251 aq->op = cfg ? NIX_AQ_INSTOP_WRITE : NIX_AQ_INSTOP_INIT; 252 253 if (rq->sso_ena) { 254 /* SSO mode */ 255 aq->rq.sso_ena = 1; 256 aq->rq.sso_tt = rq->tt; 257 aq->rq.sso_grp = rq->hwgrp; 258 aq->rq.ena_wqwd = 1; 259 aq->rq.wqe_skip = rq->wqe_skip; 260 aq->rq.wqe_caching = 1; 261 262 aq->rq.good_utag = rq->tag_mask >> 24; 263 aq->rq.bad_utag = rq->tag_mask >> 24; 264 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 265 } else { 266 /* CQ mode */ 267 aq->rq.sso_ena = 0; 268 aq->rq.good_utag = rq->tag_mask >> 24; 269 aq->rq.bad_utag = rq->tag_mask >> 24; 270 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 271 aq->rq.cq = rq->cqid; 272 } 273 274 if (rq->ipsech_ena) 275 aq->rq.ipsech_ena = 1; 276 277 aq->rq.spb_ena = 0; 278 aq->rq.lpb_aura = roc_npa_aura_handle_to_aura(rq->aura_handle); 279 280 /* Sizes must be aligned to 8 bytes */ 281 if (rq->first_skip & 0x7 || rq->later_skip & 0x7 || rq->lpb_size & 0x7) 282 return -EINVAL; 283 284 /* Expressed in number of dwords */ 285 aq->rq.first_skip = rq->first_skip / 8; 286 aq->rq.later_skip = rq->later_skip / 8; 287 aq->rq.flow_tagw = rq->flow_tag_width; /* 32-bits */ 288 aq->rq.lpb_sizem1 = rq->lpb_size / 8; 289 aq->rq.lpb_sizem1 -= 1; /* Expressed in size minus one */ 290 aq->rq.ena = ena; 291 aq->rq.pb_caching = 0x2; /* First cache aligned block to LLC */ 292 aq->rq.xqe_imm_size = 0; /* No pkt data copy to CQE */ 293 aq->rq.rq_int_ena = 0; 294 /* Many to one reduction */ 295 aq->rq.qint_idx = rq->qid % qints; 296 aq->rq.xqe_drop_ena = 1; 297 298 /* If RED enabled, then fill enable for all cases */ 299 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 300 aq->rq.spb_pool_pass = rq->spb_red_pass; 301 aq->rq.lpb_pool_pass = rq->red_pass; 302 303 aq->rq.spb_pool_drop = rq->spb_red_drop; 304 aq->rq.lpb_pool_drop = rq->red_drop; 305 } 306 307 if (cfg) { 308 if (rq->sso_ena) { 309 /* SSO mode */ 310 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 311 aq->rq_mask.sso_tt = ~aq->rq_mask.sso_tt; 312 aq->rq_mask.sso_grp = ~aq->rq_mask.sso_grp; 313 aq->rq_mask.ena_wqwd = ~aq->rq_mask.ena_wqwd; 314 aq->rq_mask.wqe_skip = ~aq->rq_mask.wqe_skip; 315 aq->rq_mask.wqe_caching = ~aq->rq_mask.wqe_caching; 316 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 317 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 318 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 319 } else { 320 /* CQ mode */ 321 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 322 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 323 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 324 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 325 aq->rq_mask.cq = ~aq->rq_mask.cq; 326 } 327 328 if (rq->ipsech_ena) 329 aq->rq_mask.ipsech_ena = ~aq->rq_mask.ipsech_ena; 330 331 aq->rq_mask.spb_ena = ~aq->rq_mask.spb_ena; 332 aq->rq_mask.lpb_aura = ~aq->rq_mask.lpb_aura; 333 aq->rq_mask.first_skip = ~aq->rq_mask.first_skip; 334 aq->rq_mask.later_skip = ~aq->rq_mask.later_skip; 335 aq->rq_mask.flow_tagw = ~aq->rq_mask.flow_tagw; 336 aq->rq_mask.lpb_sizem1 = ~aq->rq_mask.lpb_sizem1; 337 aq->rq_mask.ena = ~aq->rq_mask.ena; 338 aq->rq_mask.pb_caching = ~aq->rq_mask.pb_caching; 339 aq->rq_mask.xqe_imm_size = ~aq->rq_mask.xqe_imm_size; 340 aq->rq_mask.rq_int_ena = ~aq->rq_mask.rq_int_ena; 341 aq->rq_mask.qint_idx = ~aq->rq_mask.qint_idx; 342 aq->rq_mask.xqe_drop_ena = ~aq->rq_mask.xqe_drop_ena; 343 344 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 345 aq->rq_mask.spb_pool_pass = ~aq->rq_mask.spb_pool_pass; 346 aq->rq_mask.lpb_pool_pass = ~aq->rq_mask.lpb_pool_pass; 347 348 aq->rq_mask.spb_pool_drop = ~aq->rq_mask.spb_pool_drop; 349 aq->rq_mask.lpb_pool_drop = ~aq->rq_mask.lpb_pool_drop; 350 } 351 } 352 353 return 0; 354 } 355 356 int 357 nix_rq_cfg(struct dev *dev, struct roc_nix_rq *rq, uint16_t qints, bool cfg, 358 bool ena) 359 { 360 struct nix_cn10k_aq_enq_req *aq; 361 struct mbox *mbox = dev->mbox; 362 363 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 364 if (!aq) 365 return -ENOSPC; 366 367 aq->qidx = rq->qid; 368 aq->ctype = NIX_AQ_CTYPE_RQ; 369 aq->op = cfg ? NIX_AQ_INSTOP_WRITE : NIX_AQ_INSTOP_INIT; 370 371 if (rq->sso_ena) { 372 /* SSO mode */ 373 aq->rq.sso_ena = 1; 374 aq->rq.sso_tt = rq->tt; 375 aq->rq.sso_grp = rq->hwgrp; 376 aq->rq.ena_wqwd = 1; 377 aq->rq.wqe_skip = rq->wqe_skip; 378 aq->rq.wqe_caching = 1; 379 380 aq->rq.good_utag = rq->tag_mask >> 24; 381 aq->rq.bad_utag = rq->tag_mask >> 24; 382 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 383 384 if (rq->vwqe_ena) { 385 aq->rq.vwqe_ena = true; 386 aq->rq.vwqe_skip = rq->vwqe_first_skip; 387 /* Maximal Vector size is (2^(MAX_VSIZE_EXP+2)) */ 388 aq->rq.max_vsize_exp = rq->vwqe_max_sz_exp - 2; 389 aq->rq.vtime_wait = rq->vwqe_wait_tmo; 390 aq->rq.wqe_aura = roc_npa_aura_handle_to_aura(rq->vwqe_aura_handle); 391 } 392 } else { 393 /* CQ mode */ 394 aq->rq.sso_ena = 0; 395 aq->rq.good_utag = rq->tag_mask >> 24; 396 aq->rq.bad_utag = rq->tag_mask >> 24; 397 aq->rq.ltag = rq->tag_mask & BITMASK_ULL(24, 0); 398 aq->rq.cq = rq->cqid; 399 } 400 401 if (rq->ipsech_ena) { 402 aq->rq.ipsech_ena = 1; 403 aq->rq.ipsecd_drop_en = 1; 404 } 405 406 aq->rq.lpb_aura = roc_npa_aura_handle_to_aura(rq->aura_handle); 407 408 /* Sizes must be aligned to 8 bytes */ 409 if (rq->first_skip & 0x7 || rq->later_skip & 0x7 || rq->lpb_size & 0x7) 410 return -EINVAL; 411 412 /* Expressed in number of dwords */ 413 aq->rq.first_skip = rq->first_skip / 8; 414 aq->rq.later_skip = rq->later_skip / 8; 415 aq->rq.flow_tagw = rq->flow_tag_width; /* 32-bits */ 416 aq->rq.lpb_sizem1 = rq->lpb_size / 8; 417 aq->rq.lpb_sizem1 -= 1; /* Expressed in size minus one */ 418 aq->rq.ena = ena; 419 420 if (rq->spb_ena) { 421 uint32_t spb_sizem1; 422 423 aq->rq.spb_ena = 1; 424 aq->rq.spb_aura = 425 roc_npa_aura_handle_to_aura(rq->spb_aura_handle); 426 427 if (rq->spb_size & 0x7 || 428 rq->spb_size > NIX_RQ_CN10K_SPB_MAX_SIZE) 429 return -EINVAL; 430 431 spb_sizem1 = rq->spb_size / 8; /* Expressed in no. of dwords */ 432 spb_sizem1 -= 1; /* Expressed in size minus one */ 433 aq->rq.spb_sizem1 = spb_sizem1 & 0x3F; 434 aq->rq.spb_high_sizem1 = (spb_sizem1 >> 6) & 0x7; 435 } else { 436 aq->rq.spb_ena = 0; 437 } 438 439 aq->rq.pb_caching = 0x2; /* First cache aligned block to LLC */ 440 aq->rq.xqe_imm_size = 0; /* No pkt data copy to CQE */ 441 aq->rq.rq_int_ena = 0; 442 /* Many to one reduction */ 443 aq->rq.qint_idx = rq->qid % qints; 444 aq->rq.xqe_drop_ena = 0; 445 aq->rq.lpb_drop_ena = rq->lpb_drop_ena; 446 aq->rq.spb_drop_ena = rq->spb_drop_ena; 447 448 /* If RED enabled, then fill enable for all cases */ 449 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 450 aq->rq.spb_pool_pass = rq->spb_red_pass; 451 aq->rq.lpb_pool_pass = rq->red_pass; 452 aq->rq.wqe_pool_pass = rq->red_pass; 453 aq->rq.xqe_pass = rq->red_pass; 454 455 aq->rq.spb_pool_drop = rq->spb_red_drop; 456 aq->rq.lpb_pool_drop = rq->red_drop; 457 aq->rq.wqe_pool_drop = rq->red_drop; 458 aq->rq.xqe_drop = rq->red_drop; 459 } 460 461 if (cfg) { 462 if (rq->sso_ena) { 463 /* SSO mode */ 464 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 465 aq->rq_mask.sso_tt = ~aq->rq_mask.sso_tt; 466 aq->rq_mask.sso_grp = ~aq->rq_mask.sso_grp; 467 aq->rq_mask.ena_wqwd = ~aq->rq_mask.ena_wqwd; 468 aq->rq_mask.wqe_skip = ~aq->rq_mask.wqe_skip; 469 aq->rq_mask.wqe_caching = ~aq->rq_mask.wqe_caching; 470 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 471 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 472 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 473 if (rq->vwqe_ena) { 474 aq->rq_mask.vwqe_ena = ~aq->rq_mask.vwqe_ena; 475 aq->rq_mask.vwqe_skip = ~aq->rq_mask.vwqe_skip; 476 aq->rq_mask.max_vsize_exp = 477 ~aq->rq_mask.max_vsize_exp; 478 aq->rq_mask.vtime_wait = 479 ~aq->rq_mask.vtime_wait; 480 aq->rq_mask.wqe_aura = ~aq->rq_mask.wqe_aura; 481 } 482 } else { 483 /* CQ mode */ 484 aq->rq_mask.sso_ena = ~aq->rq_mask.sso_ena; 485 aq->rq_mask.good_utag = ~aq->rq_mask.good_utag; 486 aq->rq_mask.bad_utag = ~aq->rq_mask.bad_utag; 487 aq->rq_mask.ltag = ~aq->rq_mask.ltag; 488 aq->rq_mask.cq = ~aq->rq_mask.cq; 489 } 490 491 if (rq->ipsech_ena) 492 aq->rq_mask.ipsech_ena = ~aq->rq_mask.ipsech_ena; 493 494 if (rq->spb_ena) { 495 aq->rq_mask.spb_aura = ~aq->rq_mask.spb_aura; 496 aq->rq_mask.spb_sizem1 = ~aq->rq_mask.spb_sizem1; 497 aq->rq_mask.spb_high_sizem1 = 498 ~aq->rq_mask.spb_high_sizem1; 499 } 500 501 aq->rq_mask.spb_ena = ~aq->rq_mask.spb_ena; 502 aq->rq_mask.lpb_aura = ~aq->rq_mask.lpb_aura; 503 aq->rq_mask.first_skip = ~aq->rq_mask.first_skip; 504 aq->rq_mask.later_skip = ~aq->rq_mask.later_skip; 505 aq->rq_mask.flow_tagw = ~aq->rq_mask.flow_tagw; 506 aq->rq_mask.lpb_sizem1 = ~aq->rq_mask.lpb_sizem1; 507 aq->rq_mask.ena = ~aq->rq_mask.ena; 508 aq->rq_mask.pb_caching = ~aq->rq_mask.pb_caching; 509 aq->rq_mask.xqe_imm_size = ~aq->rq_mask.xqe_imm_size; 510 aq->rq_mask.rq_int_ena = ~aq->rq_mask.rq_int_ena; 511 aq->rq_mask.qint_idx = ~aq->rq_mask.qint_idx; 512 aq->rq_mask.xqe_drop_ena = ~aq->rq_mask.xqe_drop_ena; 513 aq->rq_mask.lpb_drop_ena = ~aq->rq_mask.lpb_drop_ena; 514 aq->rq_mask.spb_drop_ena = ~aq->rq_mask.spb_drop_ena; 515 516 if (rq->red_pass && (rq->red_pass >= rq->red_drop)) { 517 aq->rq_mask.spb_pool_pass = ~aq->rq_mask.spb_pool_pass; 518 aq->rq_mask.lpb_pool_pass = ~aq->rq_mask.lpb_pool_pass; 519 aq->rq_mask.wqe_pool_pass = ~aq->rq_mask.wqe_pool_pass; 520 aq->rq_mask.xqe_pass = ~aq->rq_mask.xqe_pass; 521 522 aq->rq_mask.spb_pool_drop = ~aq->rq_mask.spb_pool_drop; 523 aq->rq_mask.lpb_pool_drop = ~aq->rq_mask.lpb_pool_drop; 524 aq->rq_mask.wqe_pool_drop = ~aq->rq_mask.wqe_pool_drop; 525 aq->rq_mask.xqe_drop = ~aq->rq_mask.xqe_drop; 526 } 527 } 528 529 return 0; 530 } 531 532 int 533 roc_nix_rq_init(struct roc_nix *roc_nix, struct roc_nix_rq *rq, bool ena) 534 { 535 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 536 struct mbox *mbox = (&nix->dev)->mbox; 537 bool is_cn9k = roc_model_is_cn9k(); 538 struct dev *dev = &nix->dev; 539 int rc; 540 541 if (roc_nix == NULL || rq == NULL) 542 return NIX_ERR_PARAM; 543 544 if (rq->qid >= nix->nb_rx_queues) 545 return NIX_ERR_QUEUE_INVALID_RANGE; 546 547 rq->roc_nix = roc_nix; 548 549 if (is_cn9k) 550 rc = nix_rq_cn9k_cfg(dev, rq, nix->qints, false, ena); 551 else 552 rc = nix_rq_cfg(dev, rq, nix->qints, false, ena); 553 554 if (rc) 555 return rc; 556 557 rc = mbox_process(mbox); 558 if (rc) 559 return rc; 560 561 /* Update aura buf type to indicate its use */ 562 nix_rq_aura_buf_type_update(rq, true); 563 564 /* Check for meta aura if RQ is enabled */ 565 if (ena && nix->need_meta_aura) { 566 rc = roc_nix_inl_meta_aura_check(rq); 567 if (rc) 568 return rc; 569 } 570 571 return nix_tel_node_add_rq(rq); 572 } 573 574 int 575 roc_nix_rq_modify(struct roc_nix *roc_nix, struct roc_nix_rq *rq, bool ena) 576 { 577 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 578 struct mbox *mbox = (&nix->dev)->mbox; 579 bool is_cn9k = roc_model_is_cn9k(); 580 struct dev *dev = &nix->dev; 581 int rc; 582 583 if (roc_nix == NULL || rq == NULL) 584 return NIX_ERR_PARAM; 585 586 if (rq->qid >= nix->nb_rx_queues) 587 return NIX_ERR_QUEUE_INVALID_RANGE; 588 589 /* Clear attributes for existing aura's */ 590 nix_rq_aura_buf_type_update(rq, false); 591 592 rq->roc_nix = roc_nix; 593 594 if (is_cn9k) 595 rc = nix_rq_cn9k_cfg(dev, rq, nix->qints, true, ena); 596 else 597 rc = nix_rq_cfg(dev, rq, nix->qints, true, ena); 598 599 if (rc) 600 return rc; 601 602 rc = mbox_process(mbox); 603 if (rc) 604 return rc; 605 606 /* Update aura attribute to indicate its use */ 607 nix_rq_aura_buf_type_update(rq, true); 608 609 /* Check for meta aura if RQ is enabled */ 610 if (ena && nix->need_meta_aura) { 611 rc = roc_nix_inl_meta_aura_check(rq); 612 if (rc) 613 return rc; 614 } 615 616 return nix_tel_node_add_rq(rq); 617 } 618 619 int 620 roc_nix_rq_fini(struct roc_nix_rq *rq) 621 { 622 int rc; 623 624 /* Disabling RQ is sufficient */ 625 rc = roc_nix_rq_ena_dis(rq, false); 626 if (rc) 627 return rc; 628 629 /* Update aura attribute to indicate its use for */ 630 nix_rq_aura_buf_type_update(rq, false); 631 return 0; 632 } 633 634 int 635 roc_nix_cq_init(struct roc_nix *roc_nix, struct roc_nix_cq *cq) 636 { 637 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 638 struct mbox *mbox = (&nix->dev)->mbox; 639 volatile struct nix_cq_ctx_s *cq_ctx; 640 enum nix_q_size qsize; 641 size_t desc_sz; 642 int rc; 643 644 if (cq == NULL) 645 return NIX_ERR_PARAM; 646 647 qsize = nix_qsize_clampup(cq->nb_desc); 648 cq->nb_desc = nix_qsize_to_val(qsize); 649 cq->qmask = cq->nb_desc - 1; 650 cq->door = nix->base + NIX_LF_CQ_OP_DOOR; 651 cq->status = (int64_t *)(nix->base + NIX_LF_CQ_OP_STATUS); 652 cq->wdata = (uint64_t)cq->qid << 32; 653 cq->roc_nix = roc_nix; 654 655 /* CQE of W16 */ 656 desc_sz = cq->nb_desc * NIX_CQ_ENTRY_SZ; 657 cq->desc_base = plt_zmalloc(desc_sz, NIX_CQ_ALIGN); 658 if (cq->desc_base == NULL) { 659 rc = NIX_ERR_NO_MEM; 660 goto fail; 661 } 662 663 if (roc_model_is_cn9k()) { 664 struct nix_aq_enq_req *aq; 665 666 aq = mbox_alloc_msg_nix_aq_enq(mbox); 667 if (!aq) 668 return -ENOSPC; 669 670 aq->qidx = cq->qid; 671 aq->ctype = NIX_AQ_CTYPE_CQ; 672 aq->op = NIX_AQ_INSTOP_INIT; 673 cq_ctx = &aq->cq; 674 } else { 675 struct nix_cn10k_aq_enq_req *aq; 676 677 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 678 if (!aq) 679 return -ENOSPC; 680 681 aq->qidx = cq->qid; 682 aq->ctype = NIX_AQ_CTYPE_CQ; 683 aq->op = NIX_AQ_INSTOP_INIT; 684 cq_ctx = &aq->cq; 685 } 686 687 cq_ctx->ena = 1; 688 cq_ctx->caching = 1; 689 cq_ctx->qsize = qsize; 690 cq_ctx->base = (uint64_t)cq->desc_base; 691 cq_ctx->avg_level = 0xff; 692 cq_ctx->cq_err_int_ena = BIT(NIX_CQERRINT_CQE_FAULT); 693 cq_ctx->cq_err_int_ena |= BIT(NIX_CQERRINT_DOOR_ERR); 694 695 /* Many to one reduction */ 696 cq_ctx->qint_idx = cq->qid % nix->qints; 697 /* Map CQ0 [RQ0] to CINT0 and so on till max 64 irqs */ 698 cq_ctx->cint_idx = cq->qid; 699 700 if (roc_errata_nix_has_cq_min_size_4k()) { 701 const float rx_cq_skid = NIX_CQ_FULL_ERRATA_SKID; 702 uint16_t min_rx_drop; 703 704 min_rx_drop = ceil(rx_cq_skid / (float)cq->nb_desc); 705 cq_ctx->drop = min_rx_drop; 706 cq_ctx->drop_ena = 1; 707 cq->drop_thresh = min_rx_drop; 708 } else { 709 cq->drop_thresh = NIX_CQ_THRESH_LEVEL; 710 /* Drop processing or red drop cannot be enabled due to 711 * due to packets coming for second pass from CPT. 712 */ 713 if (!roc_nix_inl_inb_is_enabled(roc_nix)) { 714 cq_ctx->drop = cq->drop_thresh; 715 cq_ctx->drop_ena = 1; 716 } 717 } 718 719 /* TX pause frames enable flow ctrl on RX side */ 720 if (nix->tx_pause) { 721 /* Single BPID is allocated for all rx channels for now */ 722 cq_ctx->bpid = nix->bpid[0]; 723 cq_ctx->bp = cq->drop_thresh; 724 cq_ctx->bp_ena = 1; 725 } 726 727 rc = mbox_process(mbox); 728 if (rc) 729 goto free_mem; 730 731 return nix_tel_node_add_cq(cq); 732 733 free_mem: 734 plt_free(cq->desc_base); 735 fail: 736 return rc; 737 } 738 739 int 740 roc_nix_cq_fini(struct roc_nix_cq *cq) 741 { 742 struct mbox *mbox; 743 struct nix *nix; 744 int rc; 745 746 if (cq == NULL) 747 return NIX_ERR_PARAM; 748 749 nix = roc_nix_to_nix_priv(cq->roc_nix); 750 mbox = (&nix->dev)->mbox; 751 752 /* Disable CQ */ 753 if (roc_model_is_cn9k()) { 754 struct nix_aq_enq_req *aq; 755 756 aq = mbox_alloc_msg_nix_aq_enq(mbox); 757 if (!aq) 758 return -ENOSPC; 759 760 aq->qidx = cq->qid; 761 aq->ctype = NIX_AQ_CTYPE_CQ; 762 aq->op = NIX_AQ_INSTOP_WRITE; 763 aq->cq.ena = 0; 764 aq->cq.bp_ena = 0; 765 aq->cq_mask.ena = ~aq->cq_mask.ena; 766 aq->cq_mask.bp_ena = ~aq->cq_mask.bp_ena; 767 } else { 768 struct nix_cn10k_aq_enq_req *aq; 769 770 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 771 if (!aq) 772 return -ENOSPC; 773 774 aq->qidx = cq->qid; 775 aq->ctype = NIX_AQ_CTYPE_CQ; 776 aq->op = NIX_AQ_INSTOP_WRITE; 777 aq->cq.ena = 0; 778 aq->cq.bp_ena = 0; 779 aq->cq_mask.ena = ~aq->cq_mask.ena; 780 aq->cq_mask.bp_ena = ~aq->cq_mask.bp_ena; 781 } 782 783 rc = mbox_process(mbox); 784 if (rc) 785 return rc; 786 787 plt_free(cq->desc_base); 788 return 0; 789 } 790 791 static int 792 sqb_pool_populate(struct roc_nix *roc_nix, struct roc_nix_sq *sq) 793 { 794 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 795 uint16_t sqes_per_sqb, count, nb_sqb_bufs; 796 struct npa_pool_s pool; 797 struct npa_aura_s aura; 798 uint64_t blk_sz; 799 uint64_t iova; 800 int rc; 801 802 blk_sz = nix->sqb_size; 803 if (sq->max_sqe_sz == roc_nix_maxsqesz_w16) 804 sqes_per_sqb = (blk_sz / 8) / 16; 805 else 806 sqes_per_sqb = (blk_sz / 8) / 8; 807 808 sq->nb_desc = PLT_MAX(512U, sq->nb_desc); 809 nb_sqb_bufs = sq->nb_desc / sqes_per_sqb; 810 nb_sqb_bufs += NIX_SQB_LIST_SPACE; 811 /* Clamp up the SQB count */ 812 nb_sqb_bufs = PLT_MIN(roc_nix->max_sqb_count, 813 PLT_MAX(NIX_DEF_SQB, nb_sqb_bufs)); 814 815 sq->nb_sqb_bufs = nb_sqb_bufs; 816 sq->sqes_per_sqb_log2 = (uint16_t)plt_log2_u32(sqes_per_sqb); 817 sq->nb_sqb_bufs_adj = 818 nb_sqb_bufs - 819 (PLT_ALIGN_MUL_CEIL(nb_sqb_bufs, sqes_per_sqb) / sqes_per_sqb); 820 sq->nb_sqb_bufs_adj = 821 (sq->nb_sqb_bufs_adj * ROC_NIX_SQB_LOWER_THRESH) / 100; 822 823 nb_sqb_bufs += roc_nix->sqb_slack; 824 /* Explicitly set nat_align alone as by default pool is with both 825 * nat_align and buf_offset = 1 which we don't want for SQB. 826 */ 827 memset(&pool, 0, sizeof(struct npa_pool_s)); 828 pool.nat_align = 1; 829 830 memset(&aura, 0, sizeof(aura)); 831 aura.fc_ena = 1; 832 if (roc_model_is_cn9k() || roc_errata_npa_has_no_fc_stype_ststp()) 833 aura.fc_stype = 0x0; /* STF */ 834 else 835 aura.fc_stype = 0x3; /* STSTP */ 836 aura.fc_addr = (uint64_t)sq->fc; 837 aura.fc_hyst_bits = 0; /* Store count on all updates */ 838 rc = roc_npa_pool_create(&sq->aura_handle, blk_sz, nb_sqb_bufs, &aura, 839 &pool, 0); 840 if (rc) 841 goto fail; 842 843 roc_npa_buf_type_update(sq->aura_handle, ROC_NPA_BUF_TYPE_SQB, 1); 844 sq->sqe_mem = plt_zmalloc(blk_sz * nb_sqb_bufs, blk_sz); 845 if (sq->sqe_mem == NULL) { 846 rc = NIX_ERR_NO_MEM; 847 goto nomem; 848 } 849 850 /* Fill the initial buffers */ 851 iova = (uint64_t)sq->sqe_mem; 852 for (count = 0; count < nb_sqb_bufs; count++) { 853 roc_npa_aura_op_free(sq->aura_handle, 0, iova); 854 iova += blk_sz; 855 } 856 857 if (roc_npa_aura_op_available_wait(sq->aura_handle, nb_sqb_bufs, 0) != 858 nb_sqb_bufs) { 859 plt_err("Failed to free all pointers to the pool"); 860 rc = NIX_ERR_NO_MEM; 861 goto npa_fail; 862 } 863 864 roc_npa_aura_op_range_set(sq->aura_handle, (uint64_t)sq->sqe_mem, iova); 865 roc_npa_aura_limit_modify(sq->aura_handle, nb_sqb_bufs); 866 sq->aura_sqb_bufs = nb_sqb_bufs; 867 868 return rc; 869 npa_fail: 870 plt_free(sq->sqe_mem); 871 nomem: 872 roc_npa_pool_destroy(sq->aura_handle); 873 fail: 874 return rc; 875 } 876 877 static int 878 sq_cn9k_init(struct nix *nix, struct roc_nix_sq *sq, uint32_t rr_quantum, 879 uint16_t smq) 880 { 881 struct roc_nix *roc_nix = nix_priv_to_roc_nix(nix); 882 struct mbox *mbox = (&nix->dev)->mbox; 883 struct nix_aq_enq_req *aq; 884 885 aq = mbox_alloc_msg_nix_aq_enq(mbox); 886 if (!aq) 887 return -ENOSPC; 888 889 aq->qidx = sq->qid; 890 aq->ctype = NIX_AQ_CTYPE_SQ; 891 aq->op = NIX_AQ_INSTOP_INIT; 892 aq->sq.max_sqe_size = sq->max_sqe_sz; 893 894 aq->sq.max_sqe_size = sq->max_sqe_sz; 895 aq->sq.smq = smq; 896 aq->sq.smq_rr_quantum = rr_quantum; 897 if (roc_nix_is_sdp(roc_nix)) 898 aq->sq.default_chan = 899 nix->tx_chan_base + (sq->qid % nix->tx_chan_cnt); 900 else 901 aq->sq.default_chan = nix->tx_chan_base; 902 aq->sq.sqe_stype = NIX_STYPE_STF; 903 aq->sq.ena = 1; 904 aq->sq.sso_ena = !!sq->sso_ena; 905 aq->sq.cq_ena = !!sq->cq_ena; 906 aq->sq.cq = sq->cqid; 907 aq->sq.cq_limit = sq->cq_drop_thresh; 908 if (aq->sq.max_sqe_size == NIX_MAXSQESZ_W8) 909 aq->sq.sqe_stype = NIX_STYPE_STP; 910 aq->sq.sqb_aura = roc_npa_aura_handle_to_aura(sq->aura_handle); 911 aq->sq.sq_int_ena = BIT(NIX_SQINT_LMT_ERR); 912 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SQB_ALLOC_FAIL); 913 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SEND_ERR); 914 aq->sq.sq_int_ena |= BIT(NIX_SQINT_MNQ_ERR); 915 916 /* Many to one reduction */ 917 /* Assigning QINT 0 to all the SQs, an errata exists where NIXTX can 918 * send incorrect QINT_IDX when reporting queue interrupt (QINT). This 919 * might result in software missing the interrupt. 920 */ 921 aq->sq.qint_idx = 0; 922 return 0; 923 } 924 925 static int 926 sq_cn9k_fini(struct nix *nix, struct roc_nix_sq *sq) 927 { 928 struct mbox *mbox = (&nix->dev)->mbox; 929 struct nix_aq_enq_rsp *rsp; 930 struct nix_aq_enq_req *aq; 931 uint16_t sqes_per_sqb; 932 void *sqb_buf; 933 int rc, count; 934 935 aq = mbox_alloc_msg_nix_aq_enq(mbox); 936 if (!aq) 937 return -ENOSPC; 938 939 aq->qidx = sq->qid; 940 aq->ctype = NIX_AQ_CTYPE_SQ; 941 aq->op = NIX_AQ_INSTOP_READ; 942 rc = mbox_process_msg(mbox, (void *)&rsp); 943 if (rc) 944 return rc; 945 946 /* Check if sq is already cleaned up */ 947 if (!rsp->sq.ena) 948 return 0; 949 950 /* Disable sq */ 951 aq = mbox_alloc_msg_nix_aq_enq(mbox); 952 if (!aq) 953 return -ENOSPC; 954 955 aq->qidx = sq->qid; 956 aq->ctype = NIX_AQ_CTYPE_SQ; 957 aq->op = NIX_AQ_INSTOP_WRITE; 958 aq->sq_mask.ena = ~aq->sq_mask.ena; 959 aq->sq.ena = 0; 960 rc = mbox_process(mbox); 961 if (rc) 962 return rc; 963 964 /* Read SQ and free sqb's */ 965 aq = mbox_alloc_msg_nix_aq_enq(mbox); 966 if (!aq) 967 return -ENOSPC; 968 969 aq->qidx = sq->qid; 970 aq->ctype = NIX_AQ_CTYPE_SQ; 971 aq->op = NIX_AQ_INSTOP_READ; 972 rc = mbox_process_msg(mbox, (void *)&rsp); 973 if (rc) 974 return rc; 975 976 if (aq->sq.smq_pend) 977 plt_err("SQ has pending SQE's"); 978 979 count = aq->sq.sqb_count; 980 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2; 981 /* Free SQB's that are used */ 982 sqb_buf = (void *)rsp->sq.head_sqb; 983 while (count) { 984 void *next_sqb; 985 986 next_sqb = *(void **)((uintptr_t)sqb_buf + 987 (uint32_t)((sqes_per_sqb - 1) * 988 sq->max_sqe_sz)); 989 roc_npa_aura_op_free(sq->aura_handle, 1, (uint64_t)sqb_buf); 990 sqb_buf = next_sqb; 991 count--; 992 } 993 994 /* Free next to use sqb */ 995 if (rsp->sq.next_sqb) 996 roc_npa_aura_op_free(sq->aura_handle, 1, rsp->sq.next_sqb); 997 return 0; 998 } 999 1000 static int 1001 sq_init(struct nix *nix, struct roc_nix_sq *sq, uint32_t rr_quantum, 1002 uint16_t smq) 1003 { 1004 struct mbox *mbox = (&nix->dev)->mbox; 1005 struct nix_cn10k_aq_enq_req *aq; 1006 1007 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 1008 if (!aq) 1009 return -ENOSPC; 1010 1011 aq->qidx = sq->qid; 1012 aq->ctype = NIX_AQ_CTYPE_SQ; 1013 aq->op = NIX_AQ_INSTOP_INIT; 1014 aq->sq.max_sqe_size = sq->max_sqe_sz; 1015 1016 aq->sq.max_sqe_size = sq->max_sqe_sz; 1017 aq->sq.smq = smq; 1018 aq->sq.smq_rr_weight = rr_quantum; 1019 aq->sq.default_chan = nix->tx_chan_base; 1020 aq->sq.sqe_stype = NIX_STYPE_STF; 1021 aq->sq.ena = 1; 1022 aq->sq.sso_ena = !!sq->sso_ena; 1023 aq->sq.cq_ena = !!sq->cq_ena; 1024 aq->sq.cq = sq->cqid; 1025 aq->sq.cq_limit = sq->cq_drop_thresh; 1026 if (aq->sq.max_sqe_size == NIX_MAXSQESZ_W8) 1027 aq->sq.sqe_stype = NIX_STYPE_STP; 1028 aq->sq.sqb_aura = roc_npa_aura_handle_to_aura(sq->aura_handle); 1029 aq->sq.sq_int_ena = BIT(NIX_SQINT_LMT_ERR); 1030 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SQB_ALLOC_FAIL); 1031 aq->sq.sq_int_ena |= BIT(NIX_SQINT_SEND_ERR); 1032 aq->sq.sq_int_ena |= BIT(NIX_SQINT_MNQ_ERR); 1033 1034 /* Assigning QINT 0 to all the SQs, an errata exists where NIXTX can 1035 * send incorrect QINT_IDX when reporting queue interrupt (QINT). This 1036 * might result in software missing the interrupt. 1037 */ 1038 aq->sq.qint_idx = 0; 1039 return 0; 1040 } 1041 1042 static int 1043 sq_fini(struct nix *nix, struct roc_nix_sq *sq) 1044 { 1045 struct mbox *mbox = (&nix->dev)->mbox; 1046 struct nix_cn10k_aq_enq_rsp *rsp; 1047 struct nix_cn10k_aq_enq_req *aq; 1048 uint16_t sqes_per_sqb; 1049 void *sqb_buf; 1050 int rc, count; 1051 1052 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 1053 if (!aq) 1054 return -ENOSPC; 1055 1056 aq->qidx = sq->qid; 1057 aq->ctype = NIX_AQ_CTYPE_SQ; 1058 aq->op = NIX_AQ_INSTOP_READ; 1059 rc = mbox_process_msg(mbox, (void *)&rsp); 1060 if (rc) 1061 return rc; 1062 1063 /* Check if sq is already cleaned up */ 1064 if (!rsp->sq.ena) 1065 return 0; 1066 1067 /* Disable sq */ 1068 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 1069 if (!aq) 1070 return -ENOSPC; 1071 1072 aq->qidx = sq->qid; 1073 aq->ctype = NIX_AQ_CTYPE_SQ; 1074 aq->op = NIX_AQ_INSTOP_WRITE; 1075 aq->sq_mask.ena = ~aq->sq_mask.ena; 1076 aq->sq.ena = 0; 1077 rc = mbox_process(mbox); 1078 if (rc) 1079 return rc; 1080 1081 /* Read SQ and free sqb's */ 1082 aq = mbox_alloc_msg_nix_cn10k_aq_enq(mbox); 1083 if (!aq) 1084 return -ENOSPC; 1085 1086 aq->qidx = sq->qid; 1087 aq->ctype = NIX_AQ_CTYPE_SQ; 1088 aq->op = NIX_AQ_INSTOP_READ; 1089 rc = mbox_process_msg(mbox, (void *)&rsp); 1090 if (rc) 1091 return rc; 1092 1093 if (aq->sq.smq_pend) 1094 plt_err("SQ has pending SQE's"); 1095 1096 count = aq->sq.sqb_count; 1097 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2; 1098 /* Free SQB's that are used */ 1099 sqb_buf = (void *)rsp->sq.head_sqb; 1100 while (count) { 1101 void *next_sqb; 1102 1103 next_sqb = *(void **)((uintptr_t)sqb_buf + 1104 (uint32_t)((sqes_per_sqb - 1) * 1105 sq->max_sqe_sz)); 1106 roc_npa_aura_op_free(sq->aura_handle, 1, (uint64_t)sqb_buf); 1107 sqb_buf = next_sqb; 1108 count--; 1109 } 1110 1111 /* Free next to use sqb */ 1112 if (rsp->sq.next_sqb) 1113 roc_npa_aura_op_free(sq->aura_handle, 1, rsp->sq.next_sqb); 1114 return 0; 1115 } 1116 1117 int 1118 roc_nix_sq_init(struct roc_nix *roc_nix, struct roc_nix_sq *sq) 1119 { 1120 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 1121 struct mbox *mbox = (&nix->dev)->mbox; 1122 uint16_t qid, smq = UINT16_MAX; 1123 uint32_t rr_quantum = 0; 1124 int rc; 1125 1126 if (sq == NULL) 1127 return NIX_ERR_PARAM; 1128 1129 qid = sq->qid; 1130 if (qid >= nix->nb_tx_queues) 1131 return NIX_ERR_QUEUE_INVALID_RANGE; 1132 1133 sq->roc_nix = roc_nix; 1134 /* 1135 * Allocate memory for flow control updates from HW. 1136 * Alloc one cache line, so that fits all FC_STYPE modes. 1137 */ 1138 sq->fc = plt_zmalloc(ROC_ALIGN, ROC_ALIGN); 1139 if (sq->fc == NULL) { 1140 rc = NIX_ERR_NO_MEM; 1141 goto fail; 1142 } 1143 1144 rc = sqb_pool_populate(roc_nix, sq); 1145 if (rc) 1146 goto nomem; 1147 1148 rc = nix_tm_leaf_data_get(nix, sq->qid, &rr_quantum, &smq); 1149 if (rc) { 1150 rc = NIX_ERR_TM_LEAF_NODE_GET; 1151 goto nomem; 1152 } 1153 1154 /* Init SQ context */ 1155 if (roc_model_is_cn9k()) 1156 rc = sq_cn9k_init(nix, sq, rr_quantum, smq); 1157 else 1158 rc = sq_init(nix, sq, rr_quantum, smq); 1159 1160 if (rc) 1161 goto nomem; 1162 1163 rc = mbox_process(mbox); 1164 if (rc) 1165 goto nomem; 1166 1167 nix->sqs[qid] = sq; 1168 sq->io_addr = nix->base + NIX_LF_OP_SENDX(0); 1169 /* Evenly distribute LMT slot for each sq */ 1170 if (roc_model_is_cn9k()) { 1171 /* Multiple cores/SQ's can use same LMTLINE safely in CN9K */ 1172 sq->lmt_addr = (void *)(nix->lmt_base + 1173 ((qid & RVU_CN9K_LMT_SLOT_MASK) << 12)); 1174 } 1175 1176 rc = nix_tel_node_add_sq(sq); 1177 return rc; 1178 nomem: 1179 plt_free(sq->fc); 1180 fail: 1181 return rc; 1182 } 1183 1184 int 1185 roc_nix_sq_fini(struct roc_nix_sq *sq) 1186 { 1187 struct nix *nix; 1188 struct mbox *mbox; 1189 struct ndc_sync_op *ndc_req; 1190 uint16_t qid; 1191 int rc = 0; 1192 1193 if (sq == NULL) 1194 return NIX_ERR_PARAM; 1195 1196 nix = roc_nix_to_nix_priv(sq->roc_nix); 1197 mbox = (&nix->dev)->mbox; 1198 1199 qid = sq->qid; 1200 1201 rc = nix_tm_sq_flush_pre(sq); 1202 1203 /* Release SQ context */ 1204 if (roc_model_is_cn9k()) 1205 rc |= sq_cn9k_fini(roc_nix_to_nix_priv(sq->roc_nix), sq); 1206 else 1207 rc |= sq_fini(roc_nix_to_nix_priv(sq->roc_nix), sq); 1208 1209 /* Sync NDC-NIX-TX for LF */ 1210 ndc_req = mbox_alloc_msg_ndc_sync_op(mbox); 1211 if (ndc_req == NULL) 1212 return -ENOSPC; 1213 ndc_req->nix_lf_tx_sync = 1; 1214 if (mbox_process(mbox)) 1215 rc |= NIX_ERR_NDC_SYNC; 1216 1217 rc |= nix_tm_sq_flush_post(sq); 1218 1219 /* Restore limit to max SQB count that the pool was created 1220 * for aura drain to succeed. 1221 */ 1222 roc_npa_aura_limit_modify(sq->aura_handle, NIX_MAX_SQB); 1223 rc |= roc_npa_pool_destroy(sq->aura_handle); 1224 plt_free(sq->fc); 1225 plt_free(sq->sqe_mem); 1226 nix->sqs[qid] = NULL; 1227 1228 return rc; 1229 } 1230 1231 void 1232 roc_nix_cq_head_tail_get(struct roc_nix *roc_nix, uint16_t qid, uint32_t *head, 1233 uint32_t *tail) 1234 { 1235 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 1236 uint64_t reg, val; 1237 int64_t *addr; 1238 1239 if (head == NULL || tail == NULL) 1240 return; 1241 1242 reg = (((uint64_t)qid) << 32); 1243 addr = (int64_t *)(nix->base + NIX_LF_CQ_OP_STATUS); 1244 val = roc_atomic64_add_nosync(reg, addr); 1245 if (val & 1246 (BIT_ULL(NIX_CQ_OP_STAT_OP_ERR) | BIT_ULL(NIX_CQ_OP_STAT_CQ_ERR))) 1247 val = 0; 1248 1249 *tail = (uint32_t)(val & 0xFFFFF); 1250 *head = (uint32_t)((val >> 20) & 0xFFFFF); 1251 } 1252 1253 void 1254 roc_nix_sq_head_tail_get(struct roc_nix *roc_nix, uint16_t qid, uint32_t *head, 1255 uint32_t *tail) 1256 { 1257 struct nix *nix = roc_nix_to_nix_priv(roc_nix); 1258 struct roc_nix_sq *sq = nix->sqs[qid]; 1259 uint16_t sqes_per_sqb, sqb_cnt; 1260 uint64_t reg, val; 1261 int64_t *addr; 1262 1263 if (head == NULL || tail == NULL) 1264 return; 1265 1266 reg = (((uint64_t)qid) << 32); 1267 addr = (int64_t *)(nix->base + NIX_LF_SQ_OP_STATUS); 1268 val = roc_atomic64_add_nosync(reg, addr); 1269 if (val & BIT_ULL(NIX_CQ_OP_STAT_OP_ERR)) { 1270 val = 0; 1271 return; 1272 } 1273 1274 *tail = (uint32_t)((val >> 28) & 0x3F); 1275 *head = (uint32_t)((val >> 20) & 0x3F); 1276 sqb_cnt = (uint16_t)(val & 0xFFFF); 1277 1278 sqes_per_sqb = 1 << sq->sqes_per_sqb_log2; 1279 1280 /* Update tail index as per used sqb count */ 1281 *tail += (sqes_per_sqb * (sqb_cnt - 1)); 1282 } 1283