1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/likely.h" 7 #include "spdk/stdinc.h" 8 #include "spdk/nvme.h" 9 #include "spdk/thread.h" 10 #include "spdk/bdev_module.h" 11 #include "spdk/string.h" 12 #include "spdk/ftl.h" 13 #include "spdk/crc32.h" 14 15 #include "ftl_core.h" 16 #include "ftl_io.h" 17 #include "ftl_debug.h" 18 #include "ftl_internal.h" 19 #include "mngt/ftl_mngt.h" 20 21 22 size_t 23 spdk_ftl_io_size(void) 24 { 25 return sizeof(struct ftl_io); 26 } 27 28 static bool 29 ftl_shutdown_complete(struct spdk_ftl_dev *dev) 30 { 31 if (dev->num_inflight) { 32 return false; 33 } 34 35 if (!ftl_nv_cache_is_halted(&dev->nv_cache)) { 36 ftl_nv_cache_halt(&dev->nv_cache); 37 return false; 38 } 39 40 if (!ftl_nv_cache_chunks_busy(&dev->nv_cache)) { 41 return false; 42 } 43 44 if (!ftl_l2p_is_halted(dev)) { 45 ftl_l2p_halt(dev); 46 return false; 47 } 48 49 return true; 50 } 51 52 void 53 ftl_invalidate_addr(struct spdk_ftl_dev *dev, ftl_addr addr) 54 { 55 if (ftl_addr_in_nvc(dev, addr)) { 56 return; 57 } 58 } 59 60 void 61 spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *attrs) 62 { 63 attrs->num_blocks = dev->num_lbas; 64 attrs->block_size = FTL_BLOCK_SIZE; 65 attrs->num_zones = ftl_get_num_zones(dev); 66 attrs->zone_size = ftl_get_num_blocks_in_zone(dev); 67 attrs->optimum_io_size = dev->xfer_size; 68 } 69 70 static void 71 start_io(struct ftl_io *io) 72 { 73 struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(io->ioch); 74 75 io->map = ftl_mempool_get(ioch->map_pool); 76 if (spdk_unlikely(!io->map)) { 77 io->status = -ENOMEM; 78 ftl_io_complete(io); 79 return; 80 } 81 82 switch (io->type) { 83 case FTL_IO_READ: 84 case FTL_IO_WRITE: 85 case FTL_IO_UNMAP: 86 default: 87 io->status = -EOPNOTSUPP; 88 ftl_io_complete(io); 89 } 90 } 91 92 #define FTL_IO_QUEUE_BATCH 16 93 int 94 ftl_io_channel_poll(void *arg) 95 { 96 struct ftl_io_channel *ch = arg; 97 void *ios[FTL_IO_QUEUE_BATCH]; 98 uint64_t i, count; 99 100 count = spdk_ring_dequeue(ch->cq, ios, FTL_IO_QUEUE_BATCH); 101 if (count == 0) { 102 return SPDK_POLLER_IDLE; 103 } 104 105 for (i = 0; i < count; i++) { 106 struct ftl_io *io = ios[i]; 107 io->user_fn(io->cb_ctx, io->status); 108 } 109 110 return SPDK_POLLER_BUSY; 111 } 112 113 static void 114 ftl_process_io_channel(struct spdk_ftl_dev *dev, struct ftl_io_channel *ioch) 115 { 116 void *ios[FTL_IO_QUEUE_BATCH]; 117 size_t count, i; 118 119 count = spdk_ring_dequeue(ioch->sq, ios, FTL_IO_QUEUE_BATCH); 120 if (count == 0) { 121 return; 122 } 123 124 for (i = 0; i < count; i++) { 125 struct ftl_io *io = ios[i]; 126 start_io(io); 127 } 128 } 129 130 static void 131 ftl_process_io_queue(struct spdk_ftl_dev *dev) 132 { 133 struct ftl_io_channel *ioch; 134 135 TAILQ_FOREACH(ioch, &dev->ioch_queue, entry) { 136 ftl_process_io_channel(dev, ioch); 137 } 138 } 139 140 int 141 ftl_core_poller(void *ctx) 142 { 143 struct spdk_ftl_dev *dev = ctx; 144 uint64_t io_activity_total_old = dev->io_activity_total; 145 146 if (dev->halt && ftl_shutdown_complete(dev)) { 147 spdk_poller_unregister(&dev->core_poller); 148 return SPDK_POLLER_IDLE; 149 } 150 151 ftl_process_io_queue(dev); 152 ftl_nv_cache_process(dev); 153 ftl_l2p_process(dev); 154 155 if (io_activity_total_old != dev->io_activity_total) { 156 return SPDK_POLLER_BUSY; 157 } 158 159 return SPDK_POLLER_IDLE; 160 } 161 162 void *g_ftl_write_buf; 163 void *g_ftl_read_buf; 164 165 int 166 spdk_ftl_init(void) 167 { 168 g_ftl_write_buf = spdk_zmalloc(FTL_ZERO_BUFFER_SIZE, FTL_ZERO_BUFFER_SIZE, NULL, 169 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 170 if (!g_ftl_write_buf) { 171 return -ENOMEM; 172 } 173 174 g_ftl_read_buf = spdk_zmalloc(FTL_ZERO_BUFFER_SIZE, FTL_ZERO_BUFFER_SIZE, NULL, 175 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 176 if (!g_ftl_read_buf) { 177 spdk_free(g_ftl_write_buf); 178 g_ftl_write_buf = NULL; 179 return -ENOMEM; 180 } 181 return 0; 182 } 183 184 void 185 spdk_ftl_fini(void) 186 { 187 spdk_free(g_ftl_write_buf); 188 spdk_free(g_ftl_read_buf); 189 } 190 191 struct spdk_io_channel * 192 spdk_ftl_get_io_channel(struct spdk_ftl_dev *dev) 193 { 194 return spdk_get_io_channel(dev); 195 } 196 197 SPDK_LOG_REGISTER_COMPONENT(ftl_core) 198