1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Netronome Systems, Inc. 3 * All rights reserved. 4 */ 5 6 /* 7 * nfp_cpp_pcie_ops.c 8 * Authors: Vinayak Tammineedi <vinayak.tammineedi@netronome.com> 9 * 10 * Multiplexes the NFP BARs between NFP internal resources and 11 * implements the PCIe specific interface for generic CPP bus access. 12 * 13 * The BARs are managed and allocated if they are available. 14 * The generic CPP bus abstraction builds upon this BAR interface. 15 */ 16 17 #include "nfp6000_pcie.h" 18 19 #include <unistd.h> 20 #include <fcntl.h> 21 22 #include <rte_io.h> 23 24 #include "nfp_cpp.h" 25 #include "nfp_logs.h" 26 #include "nfp_target.h" 27 #include "nfp6000/nfp6000.h" 28 #include "../nfp_logs.h" 29 30 #define NFP_PCIE_BAR(_pf) (0x30000 + ((_pf) & 7) * 0xc0) 31 32 #define NFP_PCIE_BAR_PCIE2CPP_ACTION_BASEADDRESS(_x) (((_x) & 0x1f) << 16) 33 #define NFP_PCIE_BAR_PCIE2CPP_ACTION_BASEADDRESS_OF(_x) (((_x) >> 16) & 0x1f) 34 #define NFP_PCIE_BAR_PCIE2CPP_BASEADDRESS(_x) (((_x) & 0xffff) << 0) 35 #define NFP_PCIE_BAR_PCIE2CPP_BASEADDRESS_OF(_x) (((_x) >> 0) & 0xffff) 36 #define NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT(_x) (((_x) & 0x3) << 27) 37 #define NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_OF(_x) (((_x) >> 27) & 0x3) 38 #define NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_32BIT 0 39 #define NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_64BIT 1 40 #define NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_0BYTE 3 41 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE(_x) (((_x) & 0x7) << 29) 42 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_OF(_x) (((_x) >> 29) & 0x7) 43 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_FIXED 0 44 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_BULK 1 45 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_TARGET 2 46 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_GENERAL 3 47 #define NFP_PCIE_BAR_PCIE2CPP_TARGET_BASEADDRESS(_x) (((_x) & 0xf) << 23) 48 #define NFP_PCIE_BAR_PCIE2CPP_TARGET_BASEADDRESS_OF(_x) (((_x) >> 23) & 0xf) 49 #define NFP_PCIE_BAR_PCIE2CPP_TOKEN_BASEADDRESS(_x) (((_x) & 0x3) << 21) 50 #define NFP_PCIE_BAR_PCIE2CPP_TOKEN_BASEADDRESS_OF(_x) (((_x) >> 21) & 0x3) 51 52 /* 53 * Minimal size of the PCIe cfg memory we depend on being mapped, 54 * queue controller and DMA controller don't have to be covered. 55 */ 56 #define NFP_PCI_MIN_MAP_SIZE 0x080000 /* 512K */ 57 58 #define NFP_PCIE_P2C_FIXED_SIZE(bar) (1 << (bar)->bitsize) 59 #define NFP_PCIE_P2C_BULK_SIZE(bar) (1 << (bar)->bitsize) 60 #define NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(bar, x) ((x) << ((bar)->bitsize - 2)) 61 #define NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(bar, x) ((x) << ((bar)->bitsize - 4)) 62 #define NFP_PCIE_P2C_GENERAL_SIZE(bar) (1 << ((bar)->bitsize - 4)) 63 64 #define NFP_PCIE_P2C_EXPBAR_OFFSET(bar_index) ((bar_index) * 4) 65 66 struct nfp_pcie_user; 67 struct nfp6000_area_priv; 68 69 /* Describes BAR configuration and usage */ 70 struct nfp_bar { 71 struct nfp_pcie_user *nfp; /**< Backlink to owner */ 72 uint32_t barcfg; /**< BAR config CSR */ 73 uint64_t base; /**< Base CPP offset */ 74 uint64_t mask; /**< Mask of the BAR aperture (read only) */ 75 uint32_t bitsize; /**< Bit size of the BAR aperture (read only) */ 76 uint32_t index; /**< Index of the BAR */ 77 bool lock; /**< If the BAR has been locked */ 78 79 char *iomem; /**< mapped IO memory */ 80 struct rte_mem_resource *resource; /**< IOMEM resource window */ 81 }; 82 83 #define NFP_PCI_BAR_MAX (PCI_64BIT_BAR_COUNT * 8) 84 85 struct nfp_pcie_user { 86 struct rte_pci_device *pci_dev; 87 const struct nfp_dev_info *dev_info; 88 89 int lock; 90 91 /* PCI BAR management */ 92 uint32_t bars; 93 struct nfp_bar bar[NFP_PCI_BAR_MAX]; 94 95 /* Reserved BAR access */ 96 char *csr; 97 }; 98 99 /* Generic CPP bus access interface. */ 100 struct nfp6000_area_priv { 101 struct nfp_bar *bar; 102 uint32_t bar_offset; 103 104 int target; 105 int action; 106 int token; 107 uint64_t offset; 108 struct { 109 int read; 110 int write; 111 int bar; 112 } width; 113 size_t size; 114 char *iomem; 115 }; 116 117 static uint32_t 118 nfp_bar_maptype(struct nfp_bar *bar) 119 { 120 return NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_OF(bar->barcfg); 121 } 122 123 #define TARGET_WIDTH_32 4 124 #define TARGET_WIDTH_64 8 125 126 static int 127 nfp_compute_bar(const struct nfp_bar *bar, 128 uint32_t *bar_config, 129 uint64_t *bar_base, 130 int target, 131 int action, 132 int token, 133 uint64_t offset, 134 size_t size, 135 int width) 136 { 137 uint64_t mask; 138 uint32_t newcfg; 139 uint32_t bitsize; 140 141 if (target >= NFP_CPP_NUM_TARGETS) 142 return -EINVAL; 143 144 switch (width) { 145 case 8: 146 newcfg = NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT 147 (NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_64BIT); 148 break; 149 case 4: 150 newcfg = NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT 151 (NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_32BIT); 152 break; 153 case 0: 154 newcfg = NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT 155 (NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_0BYTE); 156 break; 157 default: 158 return -EINVAL; 159 } 160 161 if (action != NFP_CPP_ACTION_RW && action != 0) { 162 /* Fixed CPP mapping with specific action */ 163 mask = ~(NFP_PCIE_P2C_FIXED_SIZE(bar) - 1); 164 165 newcfg |= NFP_PCIE_BAR_PCIE2CPP_MAPTYPE 166 (NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_FIXED); 167 newcfg |= NFP_PCIE_BAR_PCIE2CPP_TARGET_BASEADDRESS(target); 168 newcfg |= NFP_PCIE_BAR_PCIE2CPP_ACTION_BASEADDRESS(action); 169 newcfg |= NFP_PCIE_BAR_PCIE2CPP_TOKEN_BASEADDRESS(token); 170 171 if ((offset & mask) != ((offset + size - 1) & mask)) 172 return -EINVAL; 173 174 offset &= mask; 175 bitsize = 40 - 16; 176 } else { 177 mask = ~(NFP_PCIE_P2C_BULK_SIZE(bar) - 1); 178 179 /* Bulk mapping */ 180 newcfg |= NFP_PCIE_BAR_PCIE2CPP_MAPTYPE 181 (NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_BULK); 182 newcfg |= NFP_PCIE_BAR_PCIE2CPP_TARGET_BASEADDRESS(target); 183 newcfg |= NFP_PCIE_BAR_PCIE2CPP_TOKEN_BASEADDRESS(token); 184 185 if ((offset & mask) != ((offset + size - 1) & mask)) 186 return -EINVAL; 187 188 offset &= mask; 189 bitsize = 40 - 21; 190 } 191 newcfg |= offset >> bitsize; 192 193 if (bar_base != NULL) 194 *bar_base = offset; 195 196 if (bar_config != NULL) 197 *bar_config = newcfg; 198 199 return 0; 200 } 201 202 static int 203 nfp_bar_write(struct nfp_pcie_user *nfp, 204 struct nfp_bar *bar, 205 uint32_t newcfg) 206 { 207 uint32_t xbar; 208 209 xbar = NFP_PCIE_P2C_EXPBAR_OFFSET(bar->index); 210 211 if (nfp->csr != NULL) { 212 rte_write32(newcfg, nfp->csr + xbar); 213 /* Readback to ensure BAR is flushed */ 214 rte_read32(nfp->csr + xbar); 215 } else { 216 xbar += nfp->dev_info->pcie_cfg_expbar_offset; 217 if (rte_pci_write_config(nfp->pci_dev, &newcfg, sizeof(uint32_t), 218 xbar) < 0) 219 return -EIO; 220 } 221 222 bar->barcfg = newcfg; 223 224 return 0; 225 } 226 227 static int 228 nfp_reconfigure_bar(struct nfp_pcie_user *nfp, 229 struct nfp_bar *bar, 230 int target, 231 int action, 232 int token, 233 uint64_t offset, 234 size_t size, 235 int width) 236 { 237 int err; 238 uint32_t newcfg; 239 uint64_t newbase; 240 241 err = nfp_compute_bar(bar, &newcfg, &newbase, target, action, 242 token, offset, size, width); 243 if (err != 0) 244 return err; 245 246 bar->base = newbase; 247 248 return nfp_bar_write(nfp, bar, newcfg); 249 } 250 251 static uint32_t 252 nfp_bitsize_calc(uint64_t mask) 253 { 254 uint64_t tmp = mask; 255 uint32_t bit_size = 0; 256 257 if (tmp == 0) 258 return 0; 259 260 for (; tmp != 0; tmp >>= 1) 261 bit_size++; 262 263 return bit_size; 264 } 265 266 static bool 267 nfp_bars_for_secondary(uint32_t index) 268 { 269 uint8_t tmp = index & 0x07; 270 271 if (tmp == 0x06 || tmp == 0x07) 272 return true; 273 else 274 return false; 275 } 276 277 /** 278 * Map all PCI bars and fetch the actual BAR configurations from the board. 279 * We assume that the BAR with the PCIe config block is already mapped. 280 * 281 * BAR0.0: Reserved for General Mapping (for MSI-X access to PCIe SRAM) 282 * BAR0.1: -- 283 * BAR0.2: -- 284 * BAR0.3: -- 285 * BAR0.4: -- 286 * BAR0.5: -- 287 * BAR0.6: -- 288 * BAR0.7: -- 289 * 290 * BAR1.0-BAR1.7: -- 291 * BAR2.0-BAR2.7: -- 292 */ 293 static int 294 nfp_enable_bars(struct nfp_pcie_user *nfp) 295 { 296 int pf; 297 uint32_t i; 298 uint8_t min_bars; 299 struct nfp_bar *bar; 300 enum rte_proc_type_t type; 301 struct rte_mem_resource *res; 302 const uint32_t barcfg_msix_general = NFP_PCIE_BAR_PCIE2CPP_MAPTYPE 303 (NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_GENERAL) | 304 NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_32BIT; 305 306 type = rte_eal_process_type(); 307 if (type == RTE_PROC_PRIMARY) 308 min_bars = 12; 309 else 310 min_bars = 4; 311 312 for (i = 0; i < RTE_DIM(nfp->bar); i++) { 313 if (i != 0) { 314 if (type == RTE_PROC_PRIMARY) { 315 if (nfp_bars_for_secondary(i)) 316 continue; 317 } else { 318 if (!nfp_bars_for_secondary(i)) 319 continue; 320 } 321 } 322 323 /* 24 NFP bars mapping into BAR0, BAR2 and BAR4 */ 324 res = &nfp->pci_dev->mem_resource[(i >> 3) * 2]; 325 326 /* Skip over BARs that are not mapped */ 327 if (res->addr != NULL) { 328 bar = &nfp->bar[i]; 329 bar->resource = res; 330 bar->barcfg = 0; 331 332 bar->nfp = nfp; 333 bar->index = i; 334 /* The resource shared by 8 bars */ 335 bar->mask = (res->len >> 3) - 1; 336 bar->bitsize = nfp_bitsize_calc(bar->mask); 337 bar->base = 0; 338 bar->lock = false; 339 bar->iomem = (char *)res->addr + 340 ((bar->index & 7) << bar->bitsize); 341 342 nfp->bars++; 343 } 344 } 345 346 if (nfp->bars < min_bars) { 347 PMD_DRV_LOG(ERR, "Not enough usable BARs found."); 348 return -EINVAL; 349 } 350 351 switch (nfp->pci_dev->id.device_id) { 352 case PCI_DEVICE_ID_NFP3800_PF_NIC: 353 pf = nfp->pci_dev->addr.function & 0x07; 354 nfp->csr = nfp->bar[0].iomem + NFP_PCIE_BAR(pf); 355 break; 356 case PCI_DEVICE_ID_NFP4000_PF_NIC: 357 case PCI_DEVICE_ID_NFP6000_PF_NIC: 358 nfp->csr = nfp->bar[0].iomem + NFP_PCIE_BAR(0); 359 break; 360 default: 361 PMD_DRV_LOG(ERR, "Unsupported device ID: %04hx!", 362 nfp->pci_dev->id.device_id); 363 return -EINVAL; 364 } 365 366 /* Configure, and lock, BAR0.0 for General Target use (MSI-X SRAM) */ 367 bar = &nfp->bar[0]; 368 bar->lock = true; 369 370 if (nfp_bar_write(nfp, bar, barcfg_msix_general) < 0) 371 return -EIO; 372 373 374 /* Reserve BAR2.0 for expansion rom mapping */ 375 if (type == RTE_PROC_PRIMARY) { 376 if (nfp->pci_dev->id.device_id == PCI_DEVICE_ID_NFP3800_PF_NIC) { 377 bar = &nfp->bar[16]; 378 if (bar != NULL) 379 bar->lock = true; 380 } 381 } 382 383 return 0; 384 } 385 386 /* Check if BAR can be used with the given parameters. */ 387 static bool 388 matching_bar_exist(struct nfp_bar *bar, 389 int target, 390 int action, 391 int token, 392 uint64_t offset, 393 size_t size, 394 int width) 395 { 396 int bar_width; 397 int bar_token; 398 int bar_target; 399 int bar_action; 400 uint32_t map_type; 401 402 bar_width = NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_OF(bar->barcfg); 403 switch (bar_width) { 404 case NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_32BIT: 405 bar_width = 4; 406 break; 407 case NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_64BIT: 408 bar_width = 8; 409 break; 410 case NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_0BYTE: 411 bar_width = 0; 412 break; 413 default: 414 bar_width = -1; 415 break; 416 } 417 418 /* Make sure to match up the width */ 419 if (bar_width != width) 420 return false; 421 422 bar_token = NFP_PCIE_BAR_PCIE2CPP_TOKEN_BASEADDRESS_OF(bar->barcfg); 423 bar_action = NFP_PCIE_BAR_PCIE2CPP_ACTION_BASEADDRESS_OF(bar->barcfg); 424 map_type = NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_OF(bar->barcfg); 425 switch (map_type) { 426 case NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_TARGET: 427 bar_token = -1; 428 /* FALLTHROUGH */ 429 case NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_BULK: 430 bar_action = NFP_CPP_ACTION_RW; 431 if (action == 0) 432 action = NFP_CPP_ACTION_RW; 433 /* FALLTHROUGH */ 434 case NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_FIXED: 435 break; 436 default: 437 /* We don't match explicit bars through the area interface */ 438 return false; 439 } 440 441 bar_target = NFP_PCIE_BAR_PCIE2CPP_TARGET_BASEADDRESS_OF(bar->barcfg); 442 if ((bar_target < 0 || bar_target == target) && 443 (bar_token < 0 || bar_token == token) && 444 bar_action == action && 445 bar->base <= offset && 446 (bar->base + (1 << bar->bitsize)) >= (offset + size)) 447 return true; 448 449 /* No match */ 450 return false; 451 } 452 453 static int 454 find_matching_bar(struct nfp_pcie_user *nfp, 455 int target, 456 int action, 457 int token, 458 uint64_t offset, 459 size_t size, 460 int width) 461 { 462 uint32_t n; 463 uint32_t index; 464 465 for (n = RTE_DIM(nfp->bar) ; n > 0; n--) { 466 index = n - 1; 467 struct nfp_bar *bar = &nfp->bar[index]; 468 469 if (bar->lock) 470 continue; 471 472 if (matching_bar_exist(bar, target, action, token, 473 offset, size, width)) 474 return index; 475 } 476 477 return -1; 478 } 479 480 /* Return EAGAIN if no resource is available */ 481 static int 482 find_unused_bar_noblock(struct nfp_pcie_user *nfp, 483 int target, 484 int action, 485 int token, 486 uint64_t offset, 487 size_t size, 488 int width) 489 { 490 int ret; 491 uint32_t n; 492 uint32_t index; 493 const struct nfp_bar *bar; 494 495 for (n = RTE_DIM(nfp->bar); n > 0; n--) { 496 index = n - 1; 497 bar = &nfp->bar[index]; 498 499 if (bar->bitsize == 0) 500 continue; 501 502 /* Just check to see if we can make it fit... */ 503 ret = nfp_compute_bar(bar, NULL, NULL, target, action, 504 token, offset, size, width); 505 if (ret != 0) 506 continue; 507 508 if (!bar->lock) 509 return index; 510 } 511 512 return -EAGAIN; 513 } 514 515 static int 516 nfp_alloc_bar(struct nfp_pcie_user *nfp, 517 struct nfp6000_area_priv *priv) 518 { 519 int ret; 520 int bar_num; 521 size_t size = priv->size; 522 int token = priv->token; 523 int target = priv->target; 524 int action = priv->action; 525 int width = priv->width.bar; 526 uint64_t offset = priv->offset; 527 528 /* Bar size should small than 16MB */ 529 if (size > (1 << 24)) 530 return -EINVAL; 531 532 bar_num = find_matching_bar(nfp, target, action, token, 533 offset, size, width); 534 if (bar_num >= 0) { 535 /* Found a perfect match. */ 536 nfp->bar[bar_num].lock = true; 537 return bar_num; 538 } 539 540 bar_num = find_unused_bar_noblock(nfp, target, action, token, 541 offset, size, width); 542 if (bar_num < 0) 543 return bar_num; 544 545 nfp->bar[bar_num].lock = true; 546 ret = nfp_reconfigure_bar(nfp, &nfp->bar[bar_num], 547 target, action, token, offset, size, width); 548 if (ret < 0) { 549 nfp->bar[bar_num].lock = false; 550 return ret; 551 } 552 553 return bar_num; 554 } 555 556 static void 557 nfp_disable_bars(struct nfp_pcie_user *nfp) 558 { 559 uint32_t i; 560 struct nfp_bar *bar; 561 562 for (i = 0; i < RTE_DIM(nfp->bar); i++) { 563 bar = &nfp->bar[i]; 564 if (bar->iomem != NULL) { 565 bar->iomem = NULL; 566 bar->lock = false; 567 } 568 } 569 } 570 571 static int 572 nfp6000_area_init(struct nfp_cpp_area *area, 573 uint32_t dest, 574 uint64_t address, 575 size_t size) 576 { 577 int pp; 578 int ret = 0; 579 uint32_t token = NFP_CPP_ID_TOKEN_of(dest); 580 uint32_t target = NFP_CPP_ID_TARGET_of(dest); 581 uint32_t action = NFP_CPP_ID_ACTION_of(dest); 582 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); 583 584 pp = nfp_target_pushpull(NFP_CPP_ID(target, action, token), address); 585 if (pp < 0) 586 return pp; 587 588 priv->width.read = PUSH_WIDTH(pp); 589 priv->width.write = PULL_WIDTH(pp); 590 591 if (priv->width.read > 0 && 592 priv->width.write > 0 && 593 priv->width.read != priv->width.write) 594 return -EINVAL; 595 596 if (priv->width.read > 0) 597 priv->width.bar = priv->width.read; 598 else 599 priv->width.bar = priv->width.write; 600 601 priv->bar = NULL; 602 603 priv->target = target; 604 priv->action = action; 605 priv->token = token; 606 priv->offset = address; 607 priv->size = size; 608 609 return ret; 610 } 611 612 static int 613 nfp6000_area_acquire(struct nfp_cpp_area *area) 614 { 615 int bar_num; 616 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); 617 struct nfp_pcie_user *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area)); 618 619 /* Already allocated. */ 620 if (priv->bar != NULL) 621 return 0; 622 623 bar_num = nfp_alloc_bar(nfp, priv); 624 if (bar_num < 0) { 625 PMD_DRV_LOG(ERR, "Failed to allocate bar %d:%d:%d:%#lx: %d.", 626 priv->target, priv->action, priv->token, 627 priv->offset, bar_num); 628 return bar_num; 629 } 630 631 priv->bar = &nfp->bar[bar_num]; 632 633 /* Calculate offset into BAR. */ 634 if (nfp_bar_maptype(priv->bar) == 635 NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_GENERAL) { 636 priv->bar_offset = priv->offset & 637 (NFP_PCIE_P2C_GENERAL_SIZE(priv->bar) - 1); 638 priv->bar_offset += NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(priv->bar, 639 priv->target); 640 priv->bar_offset += NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(priv->bar, 641 priv->token); 642 } else { 643 priv->bar_offset = priv->offset & priv->bar->mask; 644 } 645 646 /* Must have been too big. Sub-allocate. */ 647 if (priv->bar->iomem == NULL) 648 return -ENOMEM; 649 650 priv->iomem = priv->bar->iomem + priv->bar_offset; 651 652 return 0; 653 } 654 655 static void 656 nfp6000_area_release(struct nfp_cpp_area *area) 657 { 658 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); 659 660 priv->bar->lock = false; 661 priv->bar = NULL; 662 priv->iomem = NULL; 663 } 664 665 static void * 666 nfp6000_area_iomem(struct nfp_cpp_area *area) 667 { 668 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area); 669 return priv->iomem; 670 } 671 672 static int 673 nfp6000_area_read(struct nfp_cpp_area *area, 674 void *address, 675 uint32_t offset, 676 size_t length) 677 { 678 int ret; 679 size_t n; 680 int width; 681 uint32_t *wrptr32 = address; 682 uint64_t *wrptr64 = address; 683 struct nfp6000_area_priv *priv; 684 const volatile uint32_t *rdptr32; 685 const volatile uint64_t *rdptr64; 686 687 priv = nfp_cpp_area_priv(area); 688 rdptr64 = (uint64_t *)(priv->iomem + offset); 689 rdptr32 = (uint32_t *)(priv->iomem + offset); 690 691 if (offset + length > priv->size) 692 return -EFAULT; 693 694 width = priv->width.read; 695 if (width <= 0) 696 return -EINVAL; 697 698 /* MU reads via a PCIe2CPP BAR support 32bit (and other) lengths */ 699 if (priv->target == (NFP_CPP_TARGET_MU & NFP_CPP_TARGET_ID_MASK) && 700 priv->action == NFP_CPP_ACTION_RW && 701 (offset % sizeof(uint64_t) == 4 || 702 length % sizeof(uint64_t) == 4)) 703 width = TARGET_WIDTH_32; 704 705 /* Unaligned? Translate to an explicit access */ 706 if (((priv->offset + offset) & (width - 1)) != 0) { 707 PMD_DRV_LOG(ERR, "The aread_read unaligned!!!"); 708 return -EINVAL; 709 } 710 711 if (priv->bar == NULL) 712 return -EFAULT; 713 714 switch (width) { 715 case TARGET_WIDTH_32: 716 if (offset % sizeof(uint32_t) != 0 || 717 length % sizeof(uint32_t) != 0) 718 return -EINVAL; 719 720 for (n = 0; n < length; n += sizeof(uint32_t)) { 721 *wrptr32 = *rdptr32; 722 wrptr32++; 723 rdptr32++; 724 } 725 726 ret = n; 727 break; 728 case TARGET_WIDTH_64: 729 if (offset % sizeof(uint64_t) != 0 || 730 length % sizeof(uint64_t) != 0) 731 return -EINVAL; 732 733 for (n = 0; n < length; n += sizeof(uint64_t)) { 734 *wrptr64 = *rdptr64; 735 wrptr64++; 736 rdptr64++; 737 } 738 739 ret = n; 740 break; 741 default: 742 return -EINVAL; 743 } 744 745 return ret; 746 } 747 748 static int 749 nfp6000_area_write(struct nfp_cpp_area *area, 750 const void *address, 751 uint32_t offset, 752 size_t length) 753 { 754 int ret; 755 size_t n; 756 int width; 757 uint32_t *wrptr32; 758 uint64_t *wrptr64; 759 struct nfp6000_area_priv *priv; 760 const uint32_t *rdptr32 = address; 761 const uint64_t *rdptr64 = address; 762 763 priv = nfp_cpp_area_priv(area); 764 wrptr64 = (uint64_t *)(priv->iomem + offset); 765 wrptr32 = (uint32_t *)(priv->iomem + offset); 766 767 if (offset + length > priv->size) 768 return -EFAULT; 769 770 width = priv->width.write; 771 if (width <= 0) 772 return -EINVAL; 773 774 /* MU reads via a PCIe2CPP BAR support 32bit (and other) lengths */ 775 if (priv->target == (NFP_CPP_TARGET_MU & NFP_CPP_TARGET_ID_MASK) && 776 priv->action == NFP_CPP_ACTION_RW && 777 (offset % sizeof(uint64_t) == 4 || 778 length % sizeof(uint64_t) == 4)) 779 width = TARGET_WIDTH_32; 780 781 /* Unaligned? Translate to an explicit access */ 782 if (((priv->offset + offset) & (width - 1)) != 0) 783 return -EINVAL; 784 785 if (priv->bar == NULL) 786 return -EFAULT; 787 788 switch (width) { 789 case TARGET_WIDTH_32: 790 if (offset % sizeof(uint32_t) != 0 || 791 length % sizeof(uint32_t) != 0) 792 return -EINVAL; 793 794 for (n = 0; n < length; n += sizeof(uint32_t)) { 795 *wrptr32 = *rdptr32; 796 wrptr32++; 797 rdptr32++; 798 } 799 800 ret = n; 801 break; 802 case TARGET_WIDTH_64: 803 if (offset % sizeof(uint64_t) != 0 || 804 length % sizeof(uint64_t) != 0) 805 return -EINVAL; 806 807 for (n = 0; n < length; n += sizeof(uint64_t)) { 808 *wrptr64 = *rdptr64; 809 wrptr64++; 810 rdptr64++; 811 } 812 813 ret = n; 814 break; 815 default: 816 return -EINVAL; 817 } 818 819 return ret; 820 } 821 822 static int 823 nfp_acquire_process_lock(struct nfp_pcie_user *desc) 824 { 825 int rc; 826 struct flock lock; 827 char lockname[30]; 828 829 memset(&lock, 0, sizeof(lock)); 830 831 snprintf(lockname, sizeof(lockname), "/var/lock/nfp_%s", 832 desc->pci_dev->device.name); 833 desc->lock = open(lockname, O_RDWR | O_CREAT, 0666); 834 if (desc->lock < 0) 835 return desc->lock; 836 837 lock.l_type = F_WRLCK; 838 lock.l_whence = SEEK_SET; 839 rc = -1; 840 while (rc != 0) { 841 rc = fcntl(desc->lock, F_SETLKW, &lock); 842 if (rc < 0) { 843 if (errno != EAGAIN && errno != EACCES) { 844 close(desc->lock); 845 return rc; 846 } 847 } 848 } 849 850 return 0; 851 } 852 853 static int 854 nfp6000_get_dsn(struct rte_pci_device *pci_dev, 855 uint64_t *dsn) 856 { 857 off_t pos; 858 size_t len; 859 uint64_t tmp = 0; 860 861 pos = rte_pci_find_ext_capability(pci_dev, RTE_PCI_EXT_CAP_ID_DSN); 862 if (pos <= 0) { 863 PMD_DRV_LOG(ERR, "PCI_EXT_CAP_ID_DSN not found."); 864 return -ENODEV; 865 } 866 867 pos += 4; 868 len = sizeof(tmp); 869 870 if (rte_pci_read_config(pci_dev, &tmp, len, pos) < 0) { 871 PMD_DRV_LOG(ERR, "NFP get device serial number failed."); 872 return -ENOENT; 873 } 874 875 *dsn = tmp; 876 877 return 0; 878 } 879 880 static int 881 nfp6000_get_interface(struct rte_pci_device *dev, 882 uint16_t *interface) 883 { 884 int ret; 885 uint64_t dsn = 0; 886 887 ret = nfp6000_get_dsn(dev, &dsn); 888 if (ret != 0) 889 return ret; 890 891 *interface = dsn & 0xffff; 892 893 return 0; 894 } 895 896 static int 897 nfp6000_get_serial(struct rte_pci_device *dev, 898 uint8_t *serial, 899 size_t length) 900 { 901 int ret; 902 uint64_t dsn = 0; 903 904 if (length < NFP_SERIAL_LEN) 905 return -ENOMEM; 906 907 ret = nfp6000_get_dsn(dev, &dsn); 908 if (ret != 0) 909 return ret; 910 911 serial[0] = (dsn >> 56) & 0xff; 912 serial[1] = (dsn >> 48) & 0xff; 913 serial[2] = (dsn >> 40) & 0xff; 914 serial[3] = (dsn >> 32) & 0xff; 915 serial[4] = (dsn >> 24) & 0xff; 916 serial[5] = (dsn >> 16) & 0xff; 917 918 return 0; 919 } 920 921 static int 922 nfp6000_init(struct nfp_cpp *cpp) 923 { 924 int ret = 0; 925 struct nfp_pcie_user *desc = nfp_cpp_priv(cpp); 926 927 if (rte_eal_process_type() == RTE_PROC_PRIMARY && 928 nfp_cpp_driver_need_lock(cpp)) { 929 ret = nfp_acquire_process_lock(desc); 930 if (ret != 0) 931 return -1; 932 } 933 934 ret = nfp_enable_bars(desc); 935 if (ret != 0) { 936 PMD_DRV_LOG(ERR, "Enable bars failed."); 937 return -1; 938 } 939 940 return 0; 941 } 942 943 static void 944 nfp6000_free(struct nfp_cpp *cpp) 945 { 946 struct nfp_pcie_user *desc = nfp_cpp_priv(cpp); 947 948 nfp_disable_bars(desc); 949 if (nfp_cpp_driver_need_lock(cpp)) 950 close(desc->lock); 951 free(desc); 952 } 953 954 static const struct nfp_cpp_operations nfp6000_pcie_ops = { 955 .init = nfp6000_init, 956 .free = nfp6000_free, 957 958 .area_priv_size = sizeof(struct nfp6000_area_priv), 959 960 .get_interface = nfp6000_get_interface, 961 .get_serial = nfp6000_get_serial, 962 963 .area_init = nfp6000_area_init, 964 .area_acquire = nfp6000_area_acquire, 965 .area_release = nfp6000_area_release, 966 .area_read = nfp6000_area_read, 967 .area_write = nfp6000_area_write, 968 .area_iomem = nfp6000_area_iomem, 969 }; 970 971 const struct 972 nfp_cpp_operations *nfp_cpp_transport_operations(void) 973 { 974 return &nfp6000_pcie_ops; 975 } 976 977 /** 978 * Build a NFP CPP bus from a NFP6000 PCI device 979 * 980 * @param pdev 981 * NFP6000 PCI device 982 * @param driver_lock_needed 983 * driver lock flag 984 * 985 * @return 986 * NFP CPP handle or NULL 987 */ 988 struct nfp_cpp * 989 nfp_cpp_from_nfp6000_pcie(struct rte_pci_device *pci_dev, 990 const struct nfp_dev_info *dev_info, 991 bool driver_lock_needed) 992 { 993 int ret; 994 struct nfp_cpp *cpp; 995 uint16_t interface = 0; 996 struct nfp_pcie_user *nfp; 997 998 nfp = malloc(sizeof(*nfp)); 999 if (nfp == NULL) 1000 return NULL; 1001 1002 memset(nfp, 0, sizeof(*nfp)); 1003 nfp->pci_dev = pci_dev; 1004 nfp->dev_info = dev_info; 1005 1006 ret = nfp6000_get_interface(pci_dev, &interface); 1007 if (ret != 0) { 1008 PMD_DRV_LOG(ERR, "Get interface failed."); 1009 free(nfp); 1010 return NULL; 1011 } 1012 1013 if (NFP_CPP_INTERFACE_TYPE_of(interface) != NFP_CPP_INTERFACE_TYPE_PCI) { 1014 PMD_DRV_LOG(ERR, "Interface type is not right."); 1015 free(nfp); 1016 return NULL; 1017 } 1018 1019 if (NFP_CPP_INTERFACE_CHANNEL_of(interface) != 1020 NFP_CPP_INTERFACE_CHANNEL_PEROPENER) { 1021 PMD_DRV_LOG(ERR, "Interface channel is not right."); 1022 free(nfp); 1023 return NULL; 1024 } 1025 1026 /* Probe for all the common NFP devices */ 1027 cpp = nfp_cpp_from_device_name(pci_dev, nfp, driver_lock_needed); 1028 if (cpp == NULL) { 1029 PMD_DRV_LOG(ERR, "Get cpp from operation failed."); 1030 free(nfp); 1031 return NULL; 1032 } 1033 1034 return cpp; 1035 } 1036