1 /* 2 * BSD LICENSE 3 * 4 * Copyright (C) Cavium Inc. 2017. All Right reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Cavium networks nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <stdlib.h> 34 #include <string.h> 35 #include <stdbool.h> 36 #include <stdio.h> 37 #include <unistd.h> 38 #include <fcntl.h> 39 #include <errno.h> 40 #include <sys/mman.h> 41 42 #include <rte_atomic.h> 43 #include <rte_eal.h> 44 #include <rte_pci.h> 45 #include <rte_errno.h> 46 #include <rte_memory.h> 47 #include <rte_malloc.h> 48 #include <rte_spinlock.h> 49 #include <rte_mbuf.h> 50 51 #include "octeontx_mbox.h" 52 #include "octeontx_fpavf.h" 53 54 /* FPA Mbox Message */ 55 #define IDENTIFY 0x0 56 57 #define FPA_CONFIGSET 0x1 58 #define FPA_CONFIGGET 0x2 59 #define FPA_START_COUNT 0x3 60 #define FPA_STOP_COUNT 0x4 61 #define FPA_ATTACHAURA 0x5 62 #define FPA_DETACHAURA 0x6 63 #define FPA_SETAURALVL 0x7 64 #define FPA_GETAURALVL 0x8 65 66 #define FPA_COPROC 0x1 67 68 /* fpa mbox struct */ 69 struct octeontx_mbox_fpa_cfg { 70 int aid; 71 uint64_t pool_cfg; 72 uint64_t pool_stack_base; 73 uint64_t pool_stack_end; 74 uint64_t aura_cfg; 75 }; 76 77 struct __attribute__((__packed__)) gen_req { 78 uint32_t value; 79 }; 80 81 struct __attribute__((__packed__)) idn_req { 82 uint8_t domain_id; 83 }; 84 85 struct __attribute__((__packed__)) gen_resp { 86 uint16_t domain_id; 87 uint16_t vfid; 88 }; 89 90 struct __attribute__((__packed__)) dcfg_resp { 91 uint8_t sso_count; 92 uint8_t ssow_count; 93 uint8_t fpa_count; 94 uint8_t pko_count; 95 uint8_t tim_count; 96 uint8_t net_port_count; 97 uint8_t virt_port_count; 98 }; 99 100 #define FPA_MAX_POOL 32 101 #define FPA_PF_PAGE_SZ 4096 102 103 #define FPA_LN_SIZE 128 104 #define FPA_ROUND_UP(x, size) \ 105 ((((unsigned long)(x)) + size-1) & (~(size-1))) 106 #define FPA_OBJSZ_2_CACHE_LINE(sz) (((sz) + RTE_CACHE_LINE_MASK) >> 7) 107 #define FPA_CACHE_LINE_2_OBJSZ(sz) ((sz) << 7) 108 109 #define POOL_ENA (0x1 << 0) 110 #define POOL_DIS (0x0 << 0) 111 #define POOL_SET_NAT_ALIGN (0x1 << 1) 112 #define POOL_DIS_NAT_ALIGN (0x0 << 1) 113 #define POOL_STYPE(x) (((x) & 0x1) << 2) 114 #define POOL_LTYPE(x) (((x) & 0x3) << 3) 115 #define POOL_BUF_OFFSET(x) (((x) & 0x7fffULL) << 16) 116 #define POOL_BUF_SIZE(x) (((x) & 0x7ffULL) << 32) 117 118 struct fpavf_res { 119 void *pool_stack_base; 120 void *bar0; 121 uint64_t stack_ln_ptr; 122 uint16_t domain_id; 123 uint16_t vf_id; /* gpool_id */ 124 uint16_t sz128; /* Block size in cache lines */ 125 bool is_inuse; 126 }; 127 128 struct octeontx_fpadev { 129 rte_spinlock_t lock; 130 uint8_t total_gpool_cnt; 131 struct fpavf_res pool[FPA_VF_MAX]; 132 }; 133 134 static struct octeontx_fpadev fpadev; 135 136 /* lock is taken by caller */ 137 static int 138 octeontx_fpa_gpool_alloc(unsigned int object_size) 139 { 140 struct fpavf_res *res = NULL; 141 uint16_t gpool; 142 unsigned int sz128; 143 144 sz128 = FPA_OBJSZ_2_CACHE_LINE(object_size); 145 146 for (gpool = 0; gpool < FPA_VF_MAX; gpool++) { 147 148 /* Skip VF that is not mapped Or _inuse */ 149 if ((fpadev.pool[gpool].bar0 == NULL) || 150 (fpadev.pool[gpool].is_inuse == true)) 151 continue; 152 153 res = &fpadev.pool[gpool]; 154 155 RTE_ASSERT(res->domain_id != (uint16_t)~0); 156 RTE_ASSERT(res->vf_id != (uint16_t)~0); 157 RTE_ASSERT(res->stack_ln_ptr != 0); 158 159 if (res->sz128 == 0) { 160 res->sz128 = sz128; 161 162 fpavf_log_dbg("gpool %d blk_sz %d\n", gpool, sz128); 163 return gpool; 164 } 165 } 166 167 return -ENOSPC; 168 } 169 170 /* lock is taken by caller */ 171 static __rte_always_inline uintptr_t 172 octeontx_fpa_gpool2handle(uint16_t gpool) 173 { 174 struct fpavf_res *res = NULL; 175 176 RTE_ASSERT(gpool < FPA_VF_MAX); 177 178 res = &fpadev.pool[gpool]; 179 return (uintptr_t)res->bar0 | gpool; 180 } 181 182 static __rte_always_inline bool 183 octeontx_fpa_handle_valid(uintptr_t handle) 184 { 185 struct fpavf_res *res = NULL; 186 uint8_t gpool; 187 int i; 188 bool ret = false; 189 190 if (unlikely(!handle)) 191 return ret; 192 193 /* get the gpool */ 194 gpool = octeontx_fpa_bufpool_gpool(handle); 195 196 /* get the bar address */ 197 handle &= ~(uint64_t)FPA_GPOOL_MASK; 198 for (i = 0; i < FPA_VF_MAX; i++) { 199 if ((uintptr_t)fpadev.pool[i].bar0 != handle) 200 continue; 201 202 /* validate gpool */ 203 if (gpool != i) 204 return false; 205 206 res = &fpadev.pool[i]; 207 208 if (res->sz128 == 0 || res->domain_id == (uint16_t)~0 || 209 res->stack_ln_ptr == 0) 210 ret = false; 211 else 212 ret = true; 213 break; 214 } 215 216 return ret; 217 } 218 219 static int 220 octeontx_fpapf_pool_setup(unsigned int gpool, unsigned int buf_size, 221 signed short buf_offset, unsigned int max_buf_count) 222 { 223 void *memptr = NULL; 224 phys_addr_t phys_addr; 225 unsigned int memsz; 226 struct fpavf_res *fpa = NULL; 227 uint64_t reg; 228 struct octeontx_mbox_hdr hdr; 229 struct dcfg_resp resp; 230 struct octeontx_mbox_fpa_cfg cfg; 231 int ret = -1; 232 233 fpa = &fpadev.pool[gpool]; 234 memsz = FPA_ROUND_UP(max_buf_count / fpa->stack_ln_ptr, FPA_LN_SIZE) * 235 FPA_LN_SIZE; 236 237 /* Round-up to page size */ 238 memsz = (memsz + FPA_PF_PAGE_SZ - 1) & ~(uintptr_t)(FPA_PF_PAGE_SZ-1); 239 memptr = rte_malloc(NULL, memsz, RTE_CACHE_LINE_SIZE); 240 if (memptr == NULL) { 241 ret = -ENOMEM; 242 goto err; 243 } 244 245 /* Configure stack */ 246 fpa->pool_stack_base = memptr; 247 phys_addr = rte_malloc_virt2phy(memptr); 248 249 buf_size /= FPA_LN_SIZE; 250 251 /* POOL setup */ 252 hdr.coproc = FPA_COPROC; 253 hdr.msg = FPA_CONFIGSET; 254 hdr.vfid = fpa->vf_id; 255 hdr.res_code = 0; 256 257 buf_offset /= FPA_LN_SIZE; 258 reg = POOL_BUF_SIZE(buf_size) | POOL_BUF_OFFSET(buf_offset) | 259 POOL_LTYPE(0x2) | POOL_STYPE(0) | POOL_SET_NAT_ALIGN | 260 POOL_ENA; 261 262 cfg.aid = 0; 263 cfg.pool_cfg = reg; 264 cfg.pool_stack_base = phys_addr; 265 cfg.pool_stack_end = phys_addr + memsz; 266 cfg.aura_cfg = (1 << 9); 267 268 ret = octeontx_ssovf_mbox_send(&hdr, &cfg, 269 sizeof(struct octeontx_mbox_fpa_cfg), 270 &resp, sizeof(resp)); 271 if (ret < 0) { 272 ret = -EACCES; 273 goto err; 274 } 275 276 fpavf_log_dbg(" vfid %d gpool %d aid %d pool_cfg 0x%x pool_stack_base %" PRIx64 " pool_stack_end %" PRIx64" aura_cfg %" PRIx64 "\n", 277 fpa->vf_id, gpool, cfg.aid, (unsigned int)cfg.pool_cfg, 278 cfg.pool_stack_base, cfg.pool_stack_end, cfg.aura_cfg); 279 280 /* Now pool is in_use */ 281 fpa->is_inuse = true; 282 283 err: 284 if (ret < 0) 285 rte_free(memptr); 286 287 return ret; 288 } 289 290 static int 291 octeontx_fpapf_pool_destroy(unsigned int gpool_index) 292 { 293 struct octeontx_mbox_hdr hdr; 294 struct dcfg_resp resp; 295 struct octeontx_mbox_fpa_cfg cfg; 296 struct fpavf_res *fpa = NULL; 297 int ret = -1; 298 299 fpa = &fpadev.pool[gpool_index]; 300 301 hdr.coproc = FPA_COPROC; 302 hdr.msg = FPA_CONFIGSET; 303 hdr.vfid = fpa->vf_id; 304 hdr.res_code = 0; 305 306 /* reset and free the pool */ 307 cfg.aid = 0; 308 cfg.pool_cfg = 0; 309 cfg.pool_stack_base = 0; 310 cfg.pool_stack_end = 0; 311 cfg.aura_cfg = 0; 312 313 ret = octeontx_ssovf_mbox_send(&hdr, &cfg, 314 sizeof(struct octeontx_mbox_fpa_cfg), 315 &resp, sizeof(resp)); 316 if (ret < 0) { 317 ret = -EACCES; 318 goto err; 319 } 320 321 ret = 0; 322 err: 323 /* anycase free pool stack memory */ 324 rte_free(fpa->pool_stack_base); 325 fpa->pool_stack_base = NULL; 326 return ret; 327 } 328 329 static int 330 octeontx_fpapf_aura_attach(unsigned int gpool_index) 331 { 332 struct octeontx_mbox_hdr hdr; 333 struct dcfg_resp resp; 334 struct octeontx_mbox_fpa_cfg cfg; 335 int ret = 0; 336 337 if (gpool_index >= FPA_MAX_POOL) { 338 ret = -EINVAL; 339 goto err; 340 } 341 hdr.coproc = FPA_COPROC; 342 hdr.msg = FPA_ATTACHAURA; 343 hdr.vfid = gpool_index; 344 hdr.res_code = 0; 345 memset(&cfg, 0x0, sizeof(struct octeontx_mbox_fpa_cfg)); 346 cfg.aid = gpool_index; /* gpool is guara */ 347 348 ret = octeontx_ssovf_mbox_send(&hdr, &cfg, 349 sizeof(struct octeontx_mbox_fpa_cfg), 350 &resp, sizeof(resp)); 351 if (ret < 0) { 352 fpavf_log_err("Could not attach fpa "); 353 fpavf_log_err("aura %d to pool %d. Err=%d. FuncErr=%d\n", 354 gpool_index, gpool_index, ret, hdr.res_code); 355 ret = -EACCES; 356 goto err; 357 } 358 err: 359 return ret; 360 } 361 362 static int 363 octeontx_fpapf_aura_detach(unsigned int gpool_index) 364 { 365 struct octeontx_mbox_fpa_cfg cfg = {0}; 366 struct octeontx_mbox_hdr hdr = {0}; 367 int ret = 0; 368 369 if (gpool_index >= FPA_MAX_POOL) { 370 ret = -EINVAL; 371 goto err; 372 } 373 374 cfg.aid = gpool_index; /* gpool is gaura */ 375 hdr.coproc = FPA_COPROC; 376 hdr.msg = FPA_DETACHAURA; 377 hdr.vfid = gpool_index; 378 ret = octeontx_ssovf_mbox_send(&hdr, &cfg, sizeof(cfg), NULL, 0); 379 if (ret < 0) { 380 fpavf_log_err("Couldn't detach FPA aura %d Err=%d FuncErr=%d\n", 381 gpool_index, ret, hdr.res_code); 382 ret = -EINVAL; 383 } 384 385 err: 386 return ret; 387 } 388 389 static int 390 octeontx_fpavf_pool_setup(uintptr_t handle, unsigned long memsz, 391 void *memva, uint16_t gpool) 392 { 393 uint64_t va_end; 394 395 if (unlikely(!handle)) 396 return -ENODEV; 397 398 va_end = (uintptr_t)memva + memsz; 399 va_end &= ~RTE_CACHE_LINE_MASK; 400 401 /* VHPOOL setup */ 402 fpavf_write64((uintptr_t)memva, 403 (void *)((uintptr_t)handle + 404 FPA_VF_VHPOOL_START_ADDR(gpool))); 405 fpavf_write64(va_end, 406 (void *)((uintptr_t)handle + 407 FPA_VF_VHPOOL_END_ADDR(gpool))); 408 return 0; 409 } 410 411 static int 412 octeontx_fpapf_start_count(uint16_t gpool_index) 413 { 414 int ret = 0; 415 struct octeontx_mbox_hdr hdr = {0}; 416 417 if (gpool_index >= FPA_MAX_POOL) { 418 ret = -EINVAL; 419 goto err; 420 } 421 422 hdr.coproc = FPA_COPROC; 423 hdr.msg = FPA_START_COUNT; 424 hdr.vfid = gpool_index; 425 ret = octeontx_ssovf_mbox_send(&hdr, NULL, 0, NULL, 0); 426 if (ret < 0) { 427 fpavf_log_err("Could not start buffer counting for "); 428 fpavf_log_err("FPA pool %d. Err=%d. FuncErr=%d\n", 429 gpool_index, ret, hdr.res_code); 430 ret = -EINVAL; 431 goto err; 432 } 433 434 err: 435 return ret; 436 } 437 438 static __rte_always_inline int 439 octeontx_fpavf_free(unsigned int gpool) 440 { 441 int ret = 0; 442 443 if (gpool >= FPA_MAX_POOL) { 444 ret = -EINVAL; 445 goto err; 446 } 447 448 /* Pool is free */ 449 fpadev.pool[gpool].is_inuse = false; 450 451 err: 452 return ret; 453 } 454 455 static __rte_always_inline int 456 octeontx_gpool_free(uint16_t gpool) 457 { 458 if (fpadev.pool[gpool].sz128 != 0) { 459 fpadev.pool[gpool].sz128 = 0; 460 return 0; 461 } 462 return -EINVAL; 463 } 464 465 /* 466 * Return buffer size for a given pool 467 */ 468 int 469 octeontx_fpa_bufpool_block_size(uintptr_t handle) 470 { 471 struct fpavf_res *res = NULL; 472 uint8_t gpool; 473 474 if (unlikely(!octeontx_fpa_handle_valid(handle))) 475 return -EINVAL; 476 477 /* get the gpool */ 478 gpool = octeontx_fpa_bufpool_gpool(handle); 479 res = &fpadev.pool[gpool]; 480 return FPA_CACHE_LINE_2_OBJSZ(res->sz128); 481 } 482 483 int 484 octeontx_fpa_bufpool_free_count(uintptr_t handle) 485 { 486 uint64_t cnt, limit, avail; 487 uint8_t gpool; 488 uintptr_t pool_bar; 489 490 if (unlikely(!octeontx_fpa_handle_valid(handle))) 491 return -EINVAL; 492 493 /* get the gpool */ 494 gpool = octeontx_fpa_bufpool_gpool(handle); 495 496 /* Get pool bar address from handle */ 497 pool_bar = handle & ~(uint64_t)FPA_GPOOL_MASK; 498 499 cnt = fpavf_read64((void *)((uintptr_t)pool_bar + 500 FPA_VF_VHAURA_CNT(gpool))); 501 limit = fpavf_read64((void *)((uintptr_t)pool_bar + 502 FPA_VF_VHAURA_CNT_LIMIT(gpool))); 503 504 avail = fpavf_read64((void *)((uintptr_t)pool_bar + 505 FPA_VF_VHPOOL_AVAILABLE(gpool))); 506 507 return RTE_MIN(avail, (limit - cnt)); 508 } 509 510 uintptr_t 511 octeontx_fpa_bufpool_create(unsigned int object_size, unsigned int object_count, 512 unsigned int buf_offset, char **va_start, 513 int node_id) 514 { 515 unsigned int gpool; 516 void *memva; 517 unsigned long memsz; 518 uintptr_t gpool_handle; 519 uintptr_t pool_bar; 520 int res; 521 522 RTE_SET_USED(node_id); 523 RTE_BUILD_BUG_ON(sizeof(struct rte_mbuf) > OCTEONTX_FPAVF_BUF_OFFSET); 524 525 if (unlikely(*va_start == NULL)) 526 goto error_end; 527 528 object_size = RTE_CACHE_LINE_ROUNDUP(object_size); 529 if (object_size > FPA_MAX_OBJ_SIZE) { 530 errno = EINVAL; 531 goto error_end; 532 } 533 534 rte_spinlock_lock(&fpadev.lock); 535 res = octeontx_fpa_gpool_alloc(object_size); 536 537 /* Bail if failed */ 538 if (unlikely(res < 0)) { 539 errno = res; 540 goto error_unlock; 541 } 542 543 /* get fpavf */ 544 gpool = res; 545 546 /* get pool handle */ 547 gpool_handle = octeontx_fpa_gpool2handle(gpool); 548 if (!octeontx_fpa_handle_valid(gpool_handle)) { 549 errno = ENOSPC; 550 goto error_gpool_free; 551 } 552 553 /* Get pool bar address from handle */ 554 pool_bar = gpool_handle & ~(uint64_t)FPA_GPOOL_MASK; 555 556 res = octeontx_fpapf_pool_setup(gpool, object_size, buf_offset, 557 object_count); 558 if (res < 0) { 559 errno = res; 560 goto error_gpool_free; 561 } 562 563 /* populate AURA fields */ 564 res = octeontx_fpapf_aura_attach(gpool); 565 if (res < 0) { 566 errno = res; 567 goto error_pool_destroy; 568 } 569 570 /* vf pool setup */ 571 memsz = object_size * object_count; 572 memva = *va_start; 573 res = octeontx_fpavf_pool_setup(pool_bar, memsz, memva, gpool); 574 if (res < 0) { 575 errno = res; 576 goto error_gaura_detach; 577 } 578 579 /* Release lock */ 580 rte_spinlock_unlock(&fpadev.lock); 581 582 /* populate AURA registers */ 583 fpavf_write64(object_count, (void *)((uintptr_t)pool_bar + 584 FPA_VF_VHAURA_CNT(gpool))); 585 fpavf_write64(object_count, (void *)((uintptr_t)pool_bar + 586 FPA_VF_VHAURA_CNT_LIMIT(gpool))); 587 fpavf_write64(object_count + 1, (void *)((uintptr_t)pool_bar + 588 FPA_VF_VHAURA_CNT_THRESHOLD(gpool))); 589 590 octeontx_fpapf_start_count(gpool); 591 592 return gpool_handle; 593 594 error_gaura_detach: 595 (void) octeontx_fpapf_aura_detach(gpool); 596 error_pool_destroy: 597 octeontx_fpavf_free(gpool); 598 octeontx_fpapf_pool_destroy(gpool); 599 error_gpool_free: 600 octeontx_gpool_free(gpool); 601 error_unlock: 602 rte_spinlock_unlock(&fpadev.lock); 603 error_end: 604 return (uintptr_t)NULL; 605 } 606 607 /* 608 * Destroy a buffer pool. 609 */ 610 int 611 octeontx_fpa_bufpool_destroy(uintptr_t handle, int node_id) 612 { 613 void **node, **curr, *head = NULL; 614 uint64_t sz; 615 uint64_t cnt, avail; 616 uint8_t gpool; 617 uintptr_t pool_bar; 618 int ret; 619 620 RTE_SET_USED(node_id); 621 622 /* Wait for all outstanding writes to be committed */ 623 rte_smp_wmb(); 624 625 if (unlikely(!octeontx_fpa_handle_valid(handle))) 626 return -EINVAL; 627 628 /* get the pool */ 629 gpool = octeontx_fpa_bufpool_gpool(handle); 630 631 /* Get pool bar address from handle */ 632 pool_bar = handle & ~(uint64_t)FPA_GPOOL_MASK; 633 634 /* Check for no outstanding buffers */ 635 cnt = fpavf_read64((void *)((uintptr_t)pool_bar + 636 FPA_VF_VHAURA_CNT(gpool))); 637 if (cnt) { 638 fpavf_log_dbg("buffer exist in pool cnt %" PRId64 "\n", cnt); 639 return -EBUSY; 640 } 641 642 rte_spinlock_lock(&fpadev.lock); 643 644 avail = fpavf_read64((void *)((uintptr_t)pool_bar + 645 FPA_VF_VHPOOL_AVAILABLE(gpool))); 646 647 /* Prepare to empty the entire POOL */ 648 fpavf_write64(avail, (void *)((uintptr_t)pool_bar + 649 FPA_VF_VHAURA_CNT_LIMIT(gpool))); 650 fpavf_write64(avail + 1, (void *)((uintptr_t)pool_bar + 651 FPA_VF_VHAURA_CNT_THRESHOLD(gpool))); 652 653 /* Empty the pool */ 654 /* Invalidate the POOL */ 655 octeontx_gpool_free(gpool); 656 657 /* Process all buffers in the pool */ 658 while (avail--) { 659 660 /* Yank a buffer from the pool */ 661 node = (void *)(uintptr_t) 662 fpavf_read64((void *) 663 (pool_bar + FPA_VF_VHAURA_OP_ALLOC(gpool))); 664 665 if (node == NULL) { 666 fpavf_log_err("GAURA[%u] missing %" PRIx64 " buf\n", 667 gpool, avail); 668 break; 669 } 670 671 /* Imsert it into an ordered linked list */ 672 for (curr = &head; curr[0] != NULL; curr = curr[0]) { 673 if ((uintptr_t)node <= (uintptr_t)curr[0]) 674 break; 675 } 676 node[0] = curr[0]; 677 curr[0] = node; 678 } 679 680 /* Verify the linked list to be a perfect series */ 681 sz = octeontx_fpa_bufpool_block_size(handle) << 7; 682 for (curr = head; curr != NULL && curr[0] != NULL; 683 curr = curr[0]) { 684 if (curr == curr[0] || 685 ((uintptr_t)curr != ((uintptr_t)curr[0] - sz))) { 686 fpavf_log_err("POOL# %u buf sequence err (%p vs. %p)\n", 687 gpool, curr, curr[0]); 688 } 689 } 690 691 /* Disable pool operation */ 692 fpavf_write64(~0ul, (void *)((uintptr_t)pool_bar + 693 FPA_VF_VHPOOL_START_ADDR(gpool))); 694 fpavf_write64(~0ul, (void *)((uintptr_t)pool_bar + 695 FPA_VF_VHPOOL_END_ADDR(gpool))); 696 697 (void)octeontx_fpapf_pool_destroy(gpool); 698 699 /* Deactivate the AURA */ 700 fpavf_write64(0, (void *)((uintptr_t)pool_bar + 701 FPA_VF_VHAURA_CNT_LIMIT(gpool))); 702 fpavf_write64(0, (void *)((uintptr_t)pool_bar + 703 FPA_VF_VHAURA_CNT_THRESHOLD(gpool))); 704 705 ret = octeontx_fpapf_aura_detach(gpool); 706 if (ret) { 707 fpavf_log_err("Failed to dettach gaura %u. error code=%d\n", 708 gpool, ret); 709 } 710 711 /* Free VF */ 712 (void)octeontx_fpavf_free(gpool); 713 714 rte_spinlock_unlock(&fpadev.lock); 715 return 0; 716 } 717 718 static void 719 octeontx_fpavf_setup(void) 720 { 721 uint8_t i; 722 static bool init_once; 723 724 if (!init_once) { 725 rte_spinlock_init(&fpadev.lock); 726 fpadev.total_gpool_cnt = 0; 727 728 for (i = 0; i < FPA_VF_MAX; i++) { 729 730 fpadev.pool[i].domain_id = ~0; 731 fpadev.pool[i].stack_ln_ptr = 0; 732 fpadev.pool[i].sz128 = 0; 733 fpadev.pool[i].bar0 = NULL; 734 fpadev.pool[i].pool_stack_base = NULL; 735 fpadev.pool[i].is_inuse = false; 736 } 737 init_once = 1; 738 } 739 } 740 741 static int 742 octeontx_fpavf_identify(void *bar0) 743 { 744 uint64_t val; 745 uint16_t domain_id; 746 uint16_t vf_id; 747 uint64_t stack_ln_ptr; 748 749 val = fpavf_read64((void *)((uintptr_t)bar0 + 750 FPA_VF_VHAURA_CNT_THRESHOLD(0))); 751 752 domain_id = (val >> 8) & 0xffff; 753 vf_id = (val >> 24) & 0xffff; 754 755 stack_ln_ptr = fpavf_read64((void *)((uintptr_t)bar0 + 756 FPA_VF_VHPOOL_THRESHOLD(0))); 757 if (vf_id >= FPA_VF_MAX) { 758 fpavf_log_err("vf_id(%d) greater than max vf (32)\n", vf_id); 759 return -1; 760 } 761 762 if (fpadev.pool[vf_id].is_inuse) { 763 fpavf_log_err("vf_id %d is_inuse\n", vf_id); 764 return -1; 765 } 766 767 fpadev.pool[vf_id].domain_id = domain_id; 768 fpadev.pool[vf_id].vf_id = vf_id; 769 fpadev.pool[vf_id].bar0 = bar0; 770 fpadev.pool[vf_id].stack_ln_ptr = stack_ln_ptr; 771 772 /* SUCCESS */ 773 return vf_id; 774 } 775 776 /* FPAVF pcie device aka mempool probe */ 777 static int 778 fpavf_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) 779 { 780 uint8_t *idreg; 781 int res; 782 struct fpavf_res *fpa = NULL; 783 784 RTE_SET_USED(pci_drv); 785 RTE_SET_USED(fpa); 786 787 /* For secondary processes, the primary has done all the work */ 788 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 789 return 0; 790 791 if (pci_dev->mem_resource[0].addr == NULL) { 792 fpavf_log_err("Empty bars %p ", pci_dev->mem_resource[0].addr); 793 return -ENODEV; 794 } 795 idreg = pci_dev->mem_resource[0].addr; 796 797 octeontx_fpavf_setup(); 798 799 res = octeontx_fpavf_identify(idreg); 800 if (res < 0) 801 return -1; 802 803 fpa = &fpadev.pool[res]; 804 fpadev.total_gpool_cnt++; 805 rte_wmb(); 806 807 fpavf_log_dbg("total_fpavfs %d bar0 %p domain %d vf %d stk_ln_ptr 0x%x", 808 fpadev.total_gpool_cnt, fpa->bar0, fpa->domain_id, 809 fpa->vf_id, (unsigned int)fpa->stack_ln_ptr); 810 811 return 0; 812 } 813 814 static const struct rte_pci_id pci_fpavf_map[] = { 815 { 816 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 817 PCI_DEVICE_ID_OCTEONTX_FPA_VF) 818 }, 819 { 820 .vendor_id = 0, 821 }, 822 }; 823 824 static struct rte_pci_driver pci_fpavf = { 825 .id_table = pci_fpavf_map, 826 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA, 827 .probe = fpavf_probe, 828 }; 829 830 RTE_PMD_REGISTER_PCI(octeontx_fpavf, pci_fpavf); 831