1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/likely.h" 7 #include "spdk/stdinc.h" 8 #include "spdk/nvme.h" 9 #include "spdk/thread.h" 10 #include "spdk/bdev_module.h" 11 #include "spdk/string.h" 12 #include "spdk/ftl.h" 13 #include "spdk/crc32.h" 14 15 #include "ftl_core.h" 16 #include "ftl_io.h" 17 #include "ftl_debug.h" 18 #include "ftl_internal.h" 19 #include "mngt/ftl_mngt.h" 20 #include "utils/ftl_mempool.h" 21 22 23 size_t 24 spdk_ftl_io_size(void) 25 { 26 return sizeof(struct ftl_io); 27 } 28 29 static int 30 ftl_shutdown_complete(struct spdk_ftl_dev *dev) 31 { 32 if (dev->num_inflight) { 33 return 0; 34 } 35 36 return 1; 37 } 38 39 void 40 spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *attrs) 41 { 42 attrs->num_blocks = dev->num_lbas; 43 attrs->block_size = FTL_BLOCK_SIZE; 44 attrs->num_zones = ftl_get_num_zones(dev); 45 attrs->zone_size = ftl_get_num_blocks_in_zone(dev); 46 attrs->optimum_io_size = dev->xfer_size; 47 } 48 49 static void 50 start_io(struct ftl_io *io) 51 { 52 struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(io->ioch); 53 54 io->map = ftl_mempool_get(ioch->map_pool); 55 if (spdk_unlikely(!io->map)) { 56 io->status = -ENOMEM; 57 ftl_io_complete(io); 58 return; 59 } 60 61 switch (io->type) { 62 case FTL_IO_READ: 63 case FTL_IO_WRITE: 64 case FTL_IO_UNMAP: 65 default: 66 io->status = -EOPNOTSUPP; 67 ftl_io_complete(io); 68 } 69 } 70 71 #define FTL_IO_QUEUE_BATCH 16 72 int 73 ftl_io_channel_poll(void *arg) 74 { 75 struct ftl_io_channel *ch = arg; 76 void *ios[FTL_IO_QUEUE_BATCH]; 77 uint64_t i, count; 78 79 count = spdk_ring_dequeue(ch->cq, ios, FTL_IO_QUEUE_BATCH); 80 if (count == 0) { 81 return SPDK_POLLER_IDLE; 82 } 83 84 for (i = 0; i < count; i++) { 85 struct ftl_io *io = ios[i]; 86 io->user_fn(io->cb_ctx, io->status); 87 } 88 89 return SPDK_POLLER_BUSY; 90 } 91 92 static void 93 ftl_process_io_channel(struct spdk_ftl_dev *dev, struct ftl_io_channel *ioch) 94 { 95 void *ios[FTL_IO_QUEUE_BATCH]; 96 size_t count, i; 97 98 count = spdk_ring_dequeue(ioch->sq, ios, FTL_IO_QUEUE_BATCH); 99 if (count == 0) { 100 return; 101 } 102 103 for (i = 0; i < count; i++) { 104 struct ftl_io *io = ios[i]; 105 start_io(io); 106 } 107 } 108 109 static void 110 ftl_process_io_queue(struct spdk_ftl_dev *dev) 111 { 112 struct ftl_io_channel *ioch; 113 114 TAILQ_FOREACH(ioch, &dev->ioch_queue, entry) { 115 ftl_process_io_channel(dev, ioch); 116 } 117 } 118 119 int 120 ftl_core_poller(void *ctx) 121 { 122 struct spdk_ftl_dev *dev = ctx; 123 uint64_t io_activity_total_old = dev->io_activity_total; 124 125 if (dev->halt && ftl_shutdown_complete(dev)) { 126 spdk_poller_unregister(&dev->core_poller); 127 return SPDK_POLLER_IDLE; 128 } 129 130 ftl_process_io_queue(dev); 131 132 if (io_activity_total_old != dev->io_activity_total) { 133 return SPDK_POLLER_BUSY; 134 } 135 136 return SPDK_POLLER_IDLE; 137 } 138 139 void *g_ftl_write_buf; 140 void *g_ftl_read_buf; 141 142 int 143 spdk_ftl_init(void) 144 { 145 g_ftl_write_buf = spdk_zmalloc(FTL_ZERO_BUFFER_SIZE, FTL_ZERO_BUFFER_SIZE, NULL, 146 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 147 if (!g_ftl_write_buf) { 148 return -ENOMEM; 149 } 150 151 g_ftl_read_buf = spdk_zmalloc(FTL_ZERO_BUFFER_SIZE, FTL_ZERO_BUFFER_SIZE, NULL, 152 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 153 if (!g_ftl_read_buf) { 154 spdk_free(g_ftl_write_buf); 155 g_ftl_write_buf = NULL; 156 return -ENOMEM; 157 } 158 return 0; 159 } 160 161 void 162 spdk_ftl_fini(void) 163 { 164 spdk_free(g_ftl_write_buf); 165 spdk_free(g_ftl_read_buf); 166 } 167 168 struct spdk_io_channel * 169 spdk_ftl_get_io_channel(struct spdk_ftl_dev *dev) 170 { 171 return spdk_get_io_channel(dev); 172 } 173 174 SPDK_LOG_REGISTER_COMPONENT(ftl_core) 175