1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "ioat_internal.h" 37 38 #include "spdk/env.h" 39 #include "spdk/util.h" 40 41 #include "spdk_internal/log.h" 42 43 struct ioat_driver { 44 pthread_mutex_t lock; 45 TAILQ_HEAD(, spdk_ioat_chan) attached_chans; 46 }; 47 48 static struct ioat_driver g_ioat_driver = { 49 .lock = PTHREAD_MUTEX_INITIALIZER, 50 .attached_chans = TAILQ_HEAD_INITIALIZER(g_ioat_driver.attached_chans), 51 }; 52 53 static uint64_t 54 ioat_get_chansts(struct spdk_ioat_chan *ioat) 55 { 56 return spdk_mmio_read_8(&ioat->regs->chansts); 57 } 58 59 static void 60 ioat_write_chancmp(struct spdk_ioat_chan *ioat, uint64_t addr) 61 { 62 spdk_mmio_write_8(&ioat->regs->chancmp, addr); 63 } 64 65 static void 66 ioat_write_chainaddr(struct spdk_ioat_chan *ioat, uint64_t addr) 67 { 68 spdk_mmio_write_8(&ioat->regs->chainaddr, addr); 69 } 70 71 static inline void 72 ioat_suspend(struct spdk_ioat_chan *ioat) 73 { 74 ioat->regs->chancmd = SPDK_IOAT_CHANCMD_SUSPEND; 75 } 76 77 static inline void 78 ioat_reset(struct spdk_ioat_chan *ioat) 79 { 80 ioat->regs->chancmd = SPDK_IOAT_CHANCMD_RESET; 81 } 82 83 static inline uint32_t 84 ioat_reset_pending(struct spdk_ioat_chan *ioat) 85 { 86 uint8_t cmd; 87 88 cmd = ioat->regs->chancmd; 89 return (cmd & SPDK_IOAT_CHANCMD_RESET) == SPDK_IOAT_CHANCMD_RESET; 90 } 91 92 static int 93 ioat_map_pci_bar(struct spdk_ioat_chan *ioat) 94 { 95 int regs_bar, rc; 96 void *addr; 97 uint64_t phys_addr, size; 98 99 regs_bar = 0; 100 rc = spdk_pci_device_map_bar(ioat->device, regs_bar, &addr, &phys_addr, &size); 101 if (rc != 0 || addr == NULL) { 102 SPDK_ERRLOG("pci_device_map_range failed with error code %d\n", 103 rc); 104 return -1; 105 } 106 107 ioat->regs = (volatile struct spdk_ioat_registers *)addr; 108 109 return 0; 110 } 111 112 static int 113 ioat_unmap_pci_bar(struct spdk_ioat_chan *ioat) 114 { 115 int rc = 0; 116 void *addr = (void *)ioat->regs; 117 118 if (addr) { 119 rc = spdk_pci_device_unmap_bar(ioat->device, 0, addr); 120 } 121 return rc; 122 } 123 124 125 static inline uint32_t 126 ioat_get_active(struct spdk_ioat_chan *ioat) 127 { 128 return (ioat->head - ioat->tail) & ((1 << ioat->ring_size_order) - 1); 129 } 130 131 static inline uint32_t 132 ioat_get_ring_space(struct spdk_ioat_chan *ioat) 133 { 134 return (1 << ioat->ring_size_order) - ioat_get_active(ioat) - 1; 135 } 136 137 static uint32_t 138 ioat_get_ring_index(struct spdk_ioat_chan *ioat, uint32_t index) 139 { 140 return index & ((1 << ioat->ring_size_order) - 1); 141 } 142 143 static void 144 ioat_get_ring_entry(struct spdk_ioat_chan *ioat, uint32_t index, 145 struct ioat_descriptor **desc, 146 union spdk_ioat_hw_desc **hw_desc) 147 { 148 uint32_t i = ioat_get_ring_index(ioat, index); 149 150 *desc = &ioat->ring[i]; 151 *hw_desc = &ioat->hw_ring[i]; 152 } 153 154 static void 155 ioat_submit_single(struct spdk_ioat_chan *ioat) 156 { 157 ioat->head++; 158 } 159 160 static void 161 ioat_flush(struct spdk_ioat_chan *ioat) 162 { 163 ioat->regs->dmacount = (uint16_t)ioat->head; 164 } 165 166 static struct ioat_descriptor * 167 ioat_prep_null(struct spdk_ioat_chan *ioat) 168 { 169 struct ioat_descriptor *desc; 170 union spdk_ioat_hw_desc *hw_desc; 171 172 if (ioat_get_ring_space(ioat) < 1) { 173 return NULL; 174 } 175 176 ioat_get_ring_entry(ioat, ioat->head, &desc, &hw_desc); 177 178 hw_desc->dma.u.control_raw = 0; 179 hw_desc->dma.u.control.op = SPDK_IOAT_OP_COPY; 180 hw_desc->dma.u.control.null = 1; 181 hw_desc->dma.u.control.completion_update = 1; 182 183 hw_desc->dma.size = 8; 184 hw_desc->dma.src_addr = 0; 185 hw_desc->dma.dest_addr = 0; 186 187 desc->callback_fn = NULL; 188 desc->callback_arg = NULL; 189 190 ioat_submit_single(ioat); 191 192 return desc; 193 } 194 195 static struct ioat_descriptor * 196 ioat_prep_copy(struct spdk_ioat_chan *ioat, uint64_t dst, 197 uint64_t src, uint32_t len) 198 { 199 struct ioat_descriptor *desc; 200 union spdk_ioat_hw_desc *hw_desc; 201 202 assert(len <= ioat->max_xfer_size); 203 204 if (ioat_get_ring_space(ioat) < 1) { 205 return NULL; 206 } 207 208 ioat_get_ring_entry(ioat, ioat->head, &desc, &hw_desc); 209 210 hw_desc->dma.u.control_raw = 0; 211 hw_desc->dma.u.control.op = SPDK_IOAT_OP_COPY; 212 hw_desc->dma.u.control.completion_update = 1; 213 214 hw_desc->dma.size = len; 215 hw_desc->dma.src_addr = src; 216 hw_desc->dma.dest_addr = dst; 217 218 desc->callback_fn = NULL; 219 desc->callback_arg = NULL; 220 221 ioat_submit_single(ioat); 222 223 return desc; 224 } 225 226 static struct ioat_descriptor * 227 ioat_prep_fill(struct spdk_ioat_chan *ioat, uint64_t dst, 228 uint64_t fill_pattern, uint32_t len) 229 { 230 struct ioat_descriptor *desc; 231 union spdk_ioat_hw_desc *hw_desc; 232 233 assert(len <= ioat->max_xfer_size); 234 235 if (ioat_get_ring_space(ioat) < 1) { 236 return NULL; 237 } 238 239 ioat_get_ring_entry(ioat, ioat->head, &desc, &hw_desc); 240 241 hw_desc->fill.u.control_raw = 0; 242 hw_desc->fill.u.control.op = SPDK_IOAT_OP_FILL; 243 hw_desc->fill.u.control.completion_update = 1; 244 245 hw_desc->fill.size = len; 246 hw_desc->fill.src_data = fill_pattern; 247 hw_desc->fill.dest_addr = dst; 248 249 desc->callback_fn = NULL; 250 desc->callback_arg = NULL; 251 252 ioat_submit_single(ioat); 253 254 return desc; 255 } 256 257 static int ioat_reset_hw(struct spdk_ioat_chan *ioat) 258 { 259 int timeout; 260 uint64_t status; 261 uint32_t chanerr; 262 263 status = ioat_get_chansts(ioat); 264 if (is_ioat_active(status) || is_ioat_idle(status)) { 265 ioat_suspend(ioat); 266 } 267 268 timeout = 20; /* in milliseconds */ 269 while (is_ioat_active(status) || is_ioat_idle(status)) { 270 spdk_delay_us(1000); 271 timeout--; 272 if (timeout == 0) { 273 SPDK_ERRLOG("timed out waiting for suspend\n"); 274 return -1; 275 } 276 status = ioat_get_chansts(ioat); 277 } 278 279 /* 280 * Clear any outstanding errors. 281 * CHANERR is write-1-to-clear, so write the current CHANERR bits back to reset everything. 282 */ 283 chanerr = ioat->regs->chanerr; 284 ioat->regs->chanerr = chanerr; 285 286 ioat_reset(ioat); 287 288 timeout = 20; 289 while (ioat_reset_pending(ioat)) { 290 spdk_delay_us(1000); 291 timeout--; 292 if (timeout == 0) { 293 SPDK_ERRLOG("timed out waiting for reset\n"); 294 return -1; 295 } 296 } 297 298 return 0; 299 } 300 301 static int 302 ioat_process_channel_events(struct spdk_ioat_chan *ioat) 303 { 304 struct ioat_descriptor *desc; 305 uint64_t status, completed_descriptor, hw_desc_phys_addr; 306 uint32_t tail; 307 308 if (ioat->head == ioat->tail) { 309 return 0; 310 } 311 312 status = *ioat->comp_update; 313 completed_descriptor = status & SPDK_IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK; 314 315 if (is_ioat_halted(status)) { 316 SPDK_ERRLOG("Channel halted (%x)\n", ioat->regs->chanerr); 317 return -1; 318 } 319 320 if (completed_descriptor == ioat->last_seen) { 321 return 0; 322 } 323 324 do { 325 tail = ioat_get_ring_index(ioat, ioat->tail); 326 desc = &ioat->ring[tail]; 327 328 if (desc->callback_fn) { 329 desc->callback_fn(desc->callback_arg); 330 } 331 332 hw_desc_phys_addr = desc->phys_addr; 333 ioat->tail++; 334 } while (hw_desc_phys_addr != completed_descriptor); 335 336 ioat->last_seen = hw_desc_phys_addr; 337 return 0; 338 } 339 340 static int 341 ioat_channel_destruct(struct spdk_ioat_chan *ioat) 342 { 343 ioat_unmap_pci_bar(ioat); 344 345 if (ioat->ring) { 346 free(ioat->ring); 347 } 348 349 if (ioat->hw_ring) { 350 spdk_dma_free(ioat->hw_ring); 351 } 352 353 if (ioat->comp_update) { 354 spdk_dma_free((void *)ioat->comp_update); 355 ioat->comp_update = NULL; 356 } 357 358 return 0; 359 } 360 361 static int 362 ioat_channel_start(struct spdk_ioat_chan *ioat) 363 { 364 uint8_t xfercap, version; 365 uint64_t status; 366 int i, num_descriptors; 367 uint64_t comp_update_bus_addr = 0; 368 uint64_t phys_addr; 369 370 if (ioat_map_pci_bar(ioat) != 0) { 371 SPDK_ERRLOG("ioat_map_pci_bar() failed\n"); 372 return -1; 373 } 374 375 version = ioat->regs->cbver; 376 if (version < SPDK_IOAT_VER_3_0) { 377 SPDK_ERRLOG(" unsupported IOAT version %u.%u\n", 378 version >> 4, version & 0xF); 379 return -1; 380 } 381 382 /* Always support DMA copy */ 383 ioat->dma_capabilities = SPDK_IOAT_ENGINE_COPY_SUPPORTED; 384 if (ioat->regs->dmacapability & SPDK_IOAT_DMACAP_BFILL) { 385 ioat->dma_capabilities |= SPDK_IOAT_ENGINE_FILL_SUPPORTED; 386 } 387 xfercap = ioat->regs->xfercap; 388 389 /* Only bits [4:0] are valid. */ 390 xfercap &= 0x1f; 391 if (xfercap == 0) { 392 /* 0 means 4 GB max transfer size. */ 393 ioat->max_xfer_size = 1ULL << 32; 394 } else if (xfercap < 12) { 395 /* XFERCAP must be at least 12 (4 KB) according to the spec. */ 396 SPDK_ERRLOG("invalid XFERCAP value %u\n", xfercap); 397 return -1; 398 } else { 399 ioat->max_xfer_size = 1U << xfercap; 400 } 401 402 ioat->comp_update = spdk_dma_zmalloc(sizeof(*ioat->comp_update), SPDK_IOAT_CHANCMP_ALIGN, 403 &comp_update_bus_addr); 404 if (ioat->comp_update == NULL) { 405 return -1; 406 } 407 408 ioat->ring_size_order = IOAT_DEFAULT_ORDER; 409 410 num_descriptors = 1 << ioat->ring_size_order; 411 412 ioat->ring = calloc(num_descriptors, sizeof(struct ioat_descriptor)); 413 if (!ioat->ring) { 414 return -1; 415 } 416 417 ioat->hw_ring = spdk_dma_zmalloc(num_descriptors * sizeof(union spdk_ioat_hw_desc), 64, 418 NULL); 419 if (!ioat->hw_ring) { 420 return -1; 421 } 422 423 for (i = 0; i < num_descriptors; i++) { 424 phys_addr = spdk_vtophys(&ioat->hw_ring[i]); 425 if (phys_addr == SPDK_VTOPHYS_ERROR) { 426 SPDK_ERRLOG("Failed to translate descriptor %u to physical address\n", i); 427 return -1; 428 } 429 430 ioat->ring[i].phys_addr = phys_addr; 431 ioat->hw_ring[ioat_get_ring_index(ioat, i - 1)].generic.next = phys_addr; 432 } 433 434 ioat->head = 0; 435 ioat->tail = 0; 436 ioat->last_seen = 0; 437 438 ioat_reset_hw(ioat); 439 440 ioat->regs->chanctrl = SPDK_IOAT_CHANCTRL_ANY_ERR_ABORT_EN; 441 ioat_write_chancmp(ioat, comp_update_bus_addr); 442 ioat_write_chainaddr(ioat, ioat->ring[0].phys_addr); 443 444 ioat_prep_null(ioat); 445 ioat_flush(ioat); 446 447 i = 100; 448 while (i-- > 0) { 449 spdk_delay_us(100); 450 status = ioat_get_chansts(ioat); 451 if (is_ioat_idle(status)) { 452 break; 453 } 454 } 455 456 if (is_ioat_idle(status)) { 457 ioat_process_channel_events(ioat); 458 } else { 459 SPDK_ERRLOG("could not start channel: status = %p\n error = %#x\n", 460 (void *)status, ioat->regs->chanerr); 461 return -1; 462 } 463 464 return 0; 465 } 466 467 /* Caller must hold g_ioat_driver.lock */ 468 static struct spdk_ioat_chan * 469 ioat_attach(struct spdk_pci_device *device) 470 { 471 struct spdk_ioat_chan *ioat; 472 uint32_t cmd_reg; 473 474 ioat = calloc(1, sizeof(struct spdk_ioat_chan)); 475 if (ioat == NULL) { 476 return NULL; 477 } 478 479 /* Enable PCI busmaster. */ 480 spdk_pci_device_cfg_read32(device, &cmd_reg, 4); 481 cmd_reg |= 0x4; 482 spdk_pci_device_cfg_write32(device, cmd_reg, 4); 483 484 ioat->device = device; 485 486 if (ioat_channel_start(ioat) != 0) { 487 ioat_channel_destruct(ioat); 488 free(ioat); 489 return NULL; 490 } 491 492 return ioat; 493 } 494 495 struct ioat_enum_ctx { 496 spdk_ioat_probe_cb probe_cb; 497 spdk_ioat_attach_cb attach_cb; 498 void *cb_ctx; 499 }; 500 501 /* This function must only be called while holding g_ioat_driver.lock */ 502 static int 503 ioat_enum_cb(void *ctx, struct spdk_pci_device *pci_dev) 504 { 505 struct ioat_enum_ctx *enum_ctx = ctx; 506 struct spdk_ioat_chan *ioat; 507 508 /* Verify that this device is not already attached */ 509 TAILQ_FOREACH(ioat, &g_ioat_driver.attached_chans, tailq) { 510 /* 511 * NOTE: This assumes that the PCI abstraction layer will use the same device handle 512 * across enumerations; we could compare by BDF instead if this is not true. 513 */ 514 if (pci_dev == ioat->device) { 515 return 0; 516 } 517 } 518 519 if (enum_ctx->probe_cb(enum_ctx->cb_ctx, pci_dev)) { 520 /* 521 * Since I/OAT init is relatively quick, just perform the full init during probing. 522 * If this turns out to be a bottleneck later, this can be changed to work like 523 * NVMe with a list of devices to initialize in parallel. 524 */ 525 ioat = ioat_attach(pci_dev); 526 if (ioat == NULL) { 527 SPDK_ERRLOG("ioat_attach() failed\n"); 528 return -1; 529 } 530 531 TAILQ_INSERT_TAIL(&g_ioat_driver.attached_chans, ioat, tailq); 532 533 enum_ctx->attach_cb(enum_ctx->cb_ctx, pci_dev, ioat); 534 } 535 536 return 0; 537 } 538 539 int 540 spdk_ioat_probe(void *cb_ctx, spdk_ioat_probe_cb probe_cb, spdk_ioat_attach_cb attach_cb) 541 { 542 int rc; 543 struct ioat_enum_ctx enum_ctx; 544 545 pthread_mutex_lock(&g_ioat_driver.lock); 546 547 enum_ctx.probe_cb = probe_cb; 548 enum_ctx.attach_cb = attach_cb; 549 enum_ctx.cb_ctx = cb_ctx; 550 551 rc = spdk_pci_ioat_enumerate(ioat_enum_cb, &enum_ctx); 552 553 pthread_mutex_unlock(&g_ioat_driver.lock); 554 555 return rc; 556 } 557 558 int 559 spdk_ioat_detach(struct spdk_ioat_chan *ioat) 560 { 561 struct ioat_driver *driver = &g_ioat_driver; 562 563 /* ioat should be in the free list (not registered to a thread) 564 * when calling ioat_detach(). 565 */ 566 pthread_mutex_lock(&driver->lock); 567 TAILQ_REMOVE(&driver->attached_chans, ioat, tailq); 568 pthread_mutex_unlock(&driver->lock); 569 570 ioat_channel_destruct(ioat); 571 free(ioat); 572 573 return 0; 574 } 575 576 #define _2MB_PAGE(ptr) ((ptr) & ~(0x200000 - 1)) 577 #define _2MB_OFFSET(ptr) ((ptr) & (0x200000 - 1)) 578 579 int 580 spdk_ioat_submit_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn, 581 void *dst, const void *src, uint64_t nbytes) 582 { 583 struct ioat_descriptor *last_desc; 584 uint64_t remaining, op_size; 585 uint64_t vdst, vsrc; 586 uint64_t vdst_page, vsrc_page; 587 uint64_t pdst_page, psrc_page; 588 uint32_t orig_head; 589 590 if (!ioat) { 591 return -EINVAL; 592 } 593 594 orig_head = ioat->head; 595 596 vdst = (uint64_t)dst; 597 vsrc = (uint64_t)src; 598 vdst_page = vsrc_page = 0; 599 pdst_page = psrc_page = SPDK_VTOPHYS_ERROR; 600 601 remaining = nbytes; 602 while (remaining) { 603 if (_2MB_PAGE(vsrc) != vsrc_page) { 604 vsrc_page = _2MB_PAGE(vsrc); 605 psrc_page = spdk_vtophys((void *)vsrc_page); 606 } 607 608 if (_2MB_PAGE(vdst) != vdst_page) { 609 vdst_page = _2MB_PAGE(vdst); 610 pdst_page = spdk_vtophys((void *)vdst_page); 611 } 612 op_size = remaining; 613 op_size = spdk_min(op_size, (0x200000 - _2MB_OFFSET(vsrc))); 614 op_size = spdk_min(op_size, (0x200000 - _2MB_OFFSET(vdst))); 615 op_size = spdk_min(op_size, ioat->max_xfer_size); 616 remaining -= op_size; 617 618 last_desc = ioat_prep_copy(ioat, 619 pdst_page + _2MB_OFFSET(vdst), 620 psrc_page + _2MB_OFFSET(vsrc), 621 op_size); 622 623 if (remaining == 0 || last_desc == NULL) { 624 break; 625 } 626 627 vsrc += op_size; 628 vdst += op_size; 629 630 } 631 /* Issue null descriptor for null transfer */ 632 if (nbytes == 0) { 633 last_desc = ioat_prep_null(ioat); 634 } 635 636 if (last_desc) { 637 last_desc->callback_fn = cb_fn; 638 last_desc->callback_arg = cb_arg; 639 } else { 640 /* 641 * Ran out of descriptors in the ring - reset head to leave things as they were 642 * in case we managed to fill out any descriptors. 643 */ 644 ioat->head = orig_head; 645 return -ENOMEM; 646 } 647 648 ioat_flush(ioat); 649 return 0; 650 } 651 652 int 653 spdk_ioat_submit_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn, 654 void *dst, uint64_t fill_pattern, uint64_t nbytes) 655 { 656 struct ioat_descriptor *last_desc = NULL; 657 uint64_t remaining, op_size; 658 uint64_t vdst; 659 uint32_t orig_head; 660 661 if (!ioat) { 662 return -EINVAL; 663 } 664 665 if (!(ioat->dma_capabilities & SPDK_IOAT_ENGINE_FILL_SUPPORTED)) { 666 SPDK_ERRLOG("Channel does not support memory fill\n"); 667 return -1; 668 } 669 670 orig_head = ioat->head; 671 672 vdst = (uint64_t)dst; 673 remaining = nbytes; 674 675 while (remaining) { 676 op_size = remaining; 677 op_size = spdk_min(op_size, (0x200000 - _2MB_OFFSET(vdst))); 678 op_size = spdk_min(op_size, ioat->max_xfer_size); 679 remaining -= op_size; 680 681 last_desc = ioat_prep_fill(ioat, 682 spdk_vtophys((void *)vdst), 683 fill_pattern, 684 op_size); 685 686 if (remaining == 0 || last_desc == NULL) { 687 break; 688 } 689 690 vdst += op_size; 691 } 692 693 if (last_desc) { 694 last_desc->callback_fn = cb_fn; 695 last_desc->callback_arg = cb_arg; 696 } else { 697 /* 698 * Ran out of descriptors in the ring - reset head to leave things as they were 699 * in case we managed to fill out any descriptors. 700 */ 701 ioat->head = orig_head; 702 return -ENOMEM; 703 } 704 705 ioat_flush(ioat); 706 return 0; 707 } 708 709 uint32_t 710 spdk_ioat_get_dma_capabilities(struct spdk_ioat_chan *ioat) 711 { 712 if (!ioat) { 713 return 0; 714 } 715 return ioat->dma_capabilities; 716 } 717 718 int 719 spdk_ioat_process_events(struct spdk_ioat_chan *ioat) 720 { 721 return ioat_process_channel_events(ioat); 722 } 723 724 SPDK_LOG_REGISTER_COMPONENT("ioat", SPDK_LOG_IOAT) 725