1e49ccfc8SArtur Paszkiewicz /* SPDX-License-Identifier: BSD-3-Clause
2a6dbe372Spaul luse * Copyright (C) 2018 Intel Corporation.
3e49ccfc8SArtur Paszkiewicz * All rights reserved.
4e49ccfc8SArtur Paszkiewicz */
5e49ccfc8SArtur Paszkiewicz
6e49ccfc8SArtur Paszkiewicz #include "spdk/stdinc.h"
7e49ccfc8SArtur Paszkiewicz #include "spdk/nvme.h"
8e49ccfc8SArtur Paszkiewicz #include "spdk/thread.h"
9e49ccfc8SArtur Paszkiewicz #include "spdk/string.h"
10e49ccfc8SArtur Paszkiewicz #include "spdk/likely.h"
11e49ccfc8SArtur Paszkiewicz #include "spdk/ftl.h"
12e49ccfc8SArtur Paszkiewicz #include "spdk/bdev_module.h"
13e49ccfc8SArtur Paszkiewicz #include "spdk/config.h"
14e49ccfc8SArtur Paszkiewicz
15e49ccfc8SArtur Paszkiewicz #include "ftl_core.h"
1606790f25SArtur Paszkiewicz #include "ftl_io.h"
176448f336SArtur Paszkiewicz #include "ftl_band.h"
1892b5ebe0SKozlowski Mateusz #include "ftl_debug.h"
19a68a12a4SKozlowski Mateusz #include "ftl_nv_cache.h"
2031cf6336SKozlowski Mateusz #include "ftl_writer.h"
21e49ccfc8SArtur Paszkiewicz #include "ftl_utils.h"
22e49ccfc8SArtur Paszkiewicz #include "mngt/ftl_mngt.h"
23e49ccfc8SArtur Paszkiewicz
24e49ccfc8SArtur Paszkiewicz struct ftl_dev_init_ctx {
25e49ccfc8SArtur Paszkiewicz spdk_ftl_init_fn cb_fn;
26e49ccfc8SArtur Paszkiewicz /* Callback's argument */
27e49ccfc8SArtur Paszkiewicz void *cb_arg;
28e49ccfc8SArtur Paszkiewicz };
29e49ccfc8SArtur Paszkiewicz
30e49ccfc8SArtur Paszkiewicz struct ftl_dev_free_ctx {
31e49ccfc8SArtur Paszkiewicz spdk_ftl_fn cb_fn;
32e49ccfc8SArtur Paszkiewicz /* Callback's argument */
33e49ccfc8SArtur Paszkiewicz void *cb_arg;
34e49ccfc8SArtur Paszkiewicz };
35e49ccfc8SArtur Paszkiewicz
36e49ccfc8SArtur Paszkiewicz static int
init_core_thread(struct spdk_ftl_dev * dev)37e49ccfc8SArtur Paszkiewicz init_core_thread(struct spdk_ftl_dev *dev)
38e49ccfc8SArtur Paszkiewicz {
39e49ccfc8SArtur Paszkiewicz struct spdk_cpuset cpumask = {};
40e49ccfc8SArtur Paszkiewicz
41e49ccfc8SArtur Paszkiewicz /*
42e49ccfc8SArtur Paszkiewicz * If core mask is provided create core thread on first cpu that match with the mask,
43e49ccfc8SArtur Paszkiewicz * otherwise use current user thread
44e49ccfc8SArtur Paszkiewicz */
45e49ccfc8SArtur Paszkiewicz if (dev->conf.core_mask) {
46e49ccfc8SArtur Paszkiewicz if (spdk_cpuset_parse(&cpumask, dev->conf.core_mask)) {
47e49ccfc8SArtur Paszkiewicz return -EINVAL;
48e49ccfc8SArtur Paszkiewicz }
49e49ccfc8SArtur Paszkiewicz dev->core_thread = spdk_thread_create("ftl_core_thread", &cpumask);
50e49ccfc8SArtur Paszkiewicz } else {
51e49ccfc8SArtur Paszkiewicz dev->core_thread = spdk_get_thread();
52e49ccfc8SArtur Paszkiewicz }
53e49ccfc8SArtur Paszkiewicz
54e49ccfc8SArtur Paszkiewicz if (dev->core_thread == NULL) {
55e49ccfc8SArtur Paszkiewicz FTL_ERRLOG(dev, "Cannot create thread for mask %s\n", dev->conf.core_mask);
56e49ccfc8SArtur Paszkiewicz return -ENOMEM;
57e49ccfc8SArtur Paszkiewicz }
58e49ccfc8SArtur Paszkiewicz
59e49ccfc8SArtur Paszkiewicz return 0;
60e49ccfc8SArtur Paszkiewicz }
61e49ccfc8SArtur Paszkiewicz
62e49ccfc8SArtur Paszkiewicz static void
exit_thread(void * ctx)63e49ccfc8SArtur Paszkiewicz exit_thread(void *ctx)
64e49ccfc8SArtur Paszkiewicz {
65e49ccfc8SArtur Paszkiewicz struct spdk_thread *thread = ctx;
66e49ccfc8SArtur Paszkiewicz
67e49ccfc8SArtur Paszkiewicz spdk_thread_exit(thread);
68e49ccfc8SArtur Paszkiewicz }
69e49ccfc8SArtur Paszkiewicz
70e49ccfc8SArtur Paszkiewicz static void
deinit_core_thread(struct spdk_ftl_dev * dev)71e49ccfc8SArtur Paszkiewicz deinit_core_thread(struct spdk_ftl_dev *dev)
72e49ccfc8SArtur Paszkiewicz {
73e49ccfc8SArtur Paszkiewicz if (dev->core_thread && dev->conf.core_mask) {
74e49ccfc8SArtur Paszkiewicz spdk_thread_send_msg(dev->core_thread, exit_thread,
75e49ccfc8SArtur Paszkiewicz dev->core_thread);
76e49ccfc8SArtur Paszkiewicz dev->core_thread = NULL;
77e49ccfc8SArtur Paszkiewicz }
78e49ccfc8SArtur Paszkiewicz }
79e49ccfc8SArtur Paszkiewicz
80e49ccfc8SArtur Paszkiewicz static void
free_dev(struct spdk_ftl_dev * dev)81e49ccfc8SArtur Paszkiewicz free_dev(struct spdk_ftl_dev *dev)
82e49ccfc8SArtur Paszkiewicz {
83e49ccfc8SArtur Paszkiewicz if (!dev) {
84e49ccfc8SArtur Paszkiewicz return;
85e49ccfc8SArtur Paszkiewicz }
86e49ccfc8SArtur Paszkiewicz
87e49ccfc8SArtur Paszkiewicz deinit_core_thread(dev);
88c682c789SArtur Paszkiewicz spdk_ftl_conf_deinit(&dev->conf);
89fa378811SMateusz Kozlowski ftl_properties_deinit(dev);
90e49ccfc8SArtur Paszkiewicz free(dev);
91e49ccfc8SArtur Paszkiewicz }
92e49ccfc8SArtur Paszkiewicz
93e49ccfc8SArtur Paszkiewicz static struct spdk_ftl_dev *
allocate_dev(const struct spdk_ftl_conf * conf,int * error)94e49ccfc8SArtur Paszkiewicz allocate_dev(const struct spdk_ftl_conf *conf, int *error)
95e49ccfc8SArtur Paszkiewicz {
96e49ccfc8SArtur Paszkiewicz int rc;
97e49ccfc8SArtur Paszkiewicz struct spdk_ftl_dev *dev = calloc(1, sizeof(*dev));
98e49ccfc8SArtur Paszkiewicz
99e49ccfc8SArtur Paszkiewicz if (!dev) {
100e49ccfc8SArtur Paszkiewicz FTL_ERRLOG(dev, "Cannot allocate FTL device\n");
101e49ccfc8SArtur Paszkiewicz *error = -ENOMEM;
102e49ccfc8SArtur Paszkiewicz return NULL;
103e49ccfc8SArtur Paszkiewicz }
104e49ccfc8SArtur Paszkiewicz
105fa378811SMateusz Kozlowski rc = ftl_properties_init(dev);
106fa378811SMateusz Kozlowski if (rc) {
107fa378811SMateusz Kozlowski *error = rc;
108fa378811SMateusz Kozlowski goto error;
109fa378811SMateusz Kozlowski }
110fa378811SMateusz Kozlowski
111e49ccfc8SArtur Paszkiewicz rc = ftl_conf_init_dev(dev, conf);
112e49ccfc8SArtur Paszkiewicz if (rc) {
113e49ccfc8SArtur Paszkiewicz *error = rc;
114e49ccfc8SArtur Paszkiewicz goto error;
115e49ccfc8SArtur Paszkiewicz }
116e49ccfc8SArtur Paszkiewicz
117e49ccfc8SArtur Paszkiewicz rc = init_core_thread(dev);
118e49ccfc8SArtur Paszkiewicz if (rc) {
119e49ccfc8SArtur Paszkiewicz *error = rc;
120e49ccfc8SArtur Paszkiewicz goto error;
121e49ccfc8SArtur Paszkiewicz }
122e49ccfc8SArtur Paszkiewicz
123e49ccfc8SArtur Paszkiewicz TAILQ_INIT(&dev->rd_sq);
124e49ccfc8SArtur Paszkiewicz TAILQ_INIT(&dev->wr_sq);
125*106ad379SMateusz Kozlowski TAILQ_INIT(&dev->trim_sq);
126d9a631adSArtur Paszkiewicz TAILQ_INIT(&dev->ioch_queue);
127e49ccfc8SArtur Paszkiewicz
12831cf6336SKozlowski Mateusz ftl_writer_init(dev, &dev->writer_user, SPDK_FTL_LIMIT_HIGH, FTL_BAND_TYPE_COMPACTION);
12931cf6336SKozlowski Mateusz ftl_writer_init(dev, &dev->writer_gc, SPDK_FTL_LIMIT_CRIT, FTL_BAND_TYPE_GC);
13031cf6336SKozlowski Mateusz
131e49ccfc8SArtur Paszkiewicz return dev;
132e49ccfc8SArtur Paszkiewicz error:
133e49ccfc8SArtur Paszkiewicz free_dev(dev);
134e49ccfc8SArtur Paszkiewicz return NULL;
135e49ccfc8SArtur Paszkiewicz }
136e49ccfc8SArtur Paszkiewicz
137e49ccfc8SArtur Paszkiewicz static void
dev_init_cb(struct spdk_ftl_dev * dev,void * _ctx,int status)138e49ccfc8SArtur Paszkiewicz dev_init_cb(struct spdk_ftl_dev *dev, void *_ctx, int status)
139e49ccfc8SArtur Paszkiewicz {
140e49ccfc8SArtur Paszkiewicz struct ftl_dev_init_ctx *ctx = _ctx;
141f45c0075SArtur Paszkiewicz int rc;
142e49ccfc8SArtur Paszkiewicz
143e49ccfc8SArtur Paszkiewicz if (status) {
144f45c0075SArtur Paszkiewicz if (dev->init_retry) {
145f45c0075SArtur Paszkiewicz FTL_NOTICELOG(dev, "Startup retry\n");
146f45c0075SArtur Paszkiewicz rc = spdk_ftl_dev_init(&dev->conf, ctx->cb_fn, ctx->cb_arg);
147f45c0075SArtur Paszkiewicz if (!rc) {
148f45c0075SArtur Paszkiewicz free_dev(dev);
149f45c0075SArtur Paszkiewicz free(ctx);
150f45c0075SArtur Paszkiewicz return;
151f45c0075SArtur Paszkiewicz }
152f45c0075SArtur Paszkiewicz FTL_NOTICELOG(dev, "Startup retry failed: %d\n", rc);
153f45c0075SArtur Paszkiewicz }
154f45c0075SArtur Paszkiewicz
155e49ccfc8SArtur Paszkiewicz free_dev(dev);
156e49ccfc8SArtur Paszkiewicz dev = NULL;
157e49ccfc8SArtur Paszkiewicz }
158e49ccfc8SArtur Paszkiewicz ctx->cb_fn(dev, ctx->cb_arg, status);
159e49ccfc8SArtur Paszkiewicz free(ctx);
160e49ccfc8SArtur Paszkiewicz }
161e49ccfc8SArtur Paszkiewicz
162e49ccfc8SArtur Paszkiewicz int
spdk_ftl_dev_init(const struct spdk_ftl_conf * conf,spdk_ftl_init_fn cb_fn,void * cb_arg)163e49ccfc8SArtur Paszkiewicz spdk_ftl_dev_init(const struct spdk_ftl_conf *conf, spdk_ftl_init_fn cb_fn, void *cb_arg)
164e49ccfc8SArtur Paszkiewicz {
165e49ccfc8SArtur Paszkiewicz int rc = -1;
166e49ccfc8SArtur Paszkiewicz struct ftl_dev_init_ctx *ctx;
167e49ccfc8SArtur Paszkiewicz struct spdk_ftl_dev *dev = NULL;
168e49ccfc8SArtur Paszkiewicz
169e49ccfc8SArtur Paszkiewicz ctx = calloc(1, sizeof(*ctx));
170e49ccfc8SArtur Paszkiewicz if (!ctx) {
171e49ccfc8SArtur Paszkiewicz rc = -ENOMEM;
172e49ccfc8SArtur Paszkiewicz goto error;
173e49ccfc8SArtur Paszkiewicz }
174e49ccfc8SArtur Paszkiewicz ctx->cb_fn = cb_fn;
175e49ccfc8SArtur Paszkiewicz ctx->cb_arg = cb_arg;
176e49ccfc8SArtur Paszkiewicz
177e49ccfc8SArtur Paszkiewicz dev = allocate_dev(conf, &rc);
178e49ccfc8SArtur Paszkiewicz if (!dev) {
179e49ccfc8SArtur Paszkiewicz goto error;
180e49ccfc8SArtur Paszkiewicz }
181e49ccfc8SArtur Paszkiewicz
182e49ccfc8SArtur Paszkiewicz rc = ftl_mngt_call_dev_startup(dev, dev_init_cb, ctx);
183e49ccfc8SArtur Paszkiewicz if (rc) {
184e49ccfc8SArtur Paszkiewicz goto error;
185e49ccfc8SArtur Paszkiewicz }
186e49ccfc8SArtur Paszkiewicz
187e49ccfc8SArtur Paszkiewicz return 0;
188e49ccfc8SArtur Paszkiewicz
189e49ccfc8SArtur Paszkiewicz error:
190e49ccfc8SArtur Paszkiewicz free(ctx);
191e49ccfc8SArtur Paszkiewicz free_dev(dev);
192e49ccfc8SArtur Paszkiewicz return rc;
193e49ccfc8SArtur Paszkiewicz }
194e49ccfc8SArtur Paszkiewicz
195e49ccfc8SArtur Paszkiewicz static void
dev_free_cb(struct spdk_ftl_dev * dev,void * _ctx,int status)196e49ccfc8SArtur Paszkiewicz dev_free_cb(struct spdk_ftl_dev *dev, void *_ctx, int status)
197e49ccfc8SArtur Paszkiewicz {
198e49ccfc8SArtur Paszkiewicz struct ftl_dev_free_ctx *ctx = _ctx;
199e49ccfc8SArtur Paszkiewicz
200e49ccfc8SArtur Paszkiewicz if (!status) {
201e49ccfc8SArtur Paszkiewicz free_dev(dev);
202e49ccfc8SArtur Paszkiewicz }
203e49ccfc8SArtur Paszkiewicz ctx->cb_fn(ctx->cb_arg, status);
204e49ccfc8SArtur Paszkiewicz free(ctx);
205e49ccfc8SArtur Paszkiewicz }
206e49ccfc8SArtur Paszkiewicz
207e49ccfc8SArtur Paszkiewicz int
spdk_ftl_dev_free(struct spdk_ftl_dev * dev,spdk_ftl_fn cb_fn,void * cb_arg)208e49ccfc8SArtur Paszkiewicz spdk_ftl_dev_free(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg)
209e49ccfc8SArtur Paszkiewicz {
210e49ccfc8SArtur Paszkiewicz int rc = -1;
211e49ccfc8SArtur Paszkiewicz struct ftl_dev_free_ctx *ctx;
212e49ccfc8SArtur Paszkiewicz
213e49ccfc8SArtur Paszkiewicz ctx = calloc(1, sizeof(*ctx));
214e49ccfc8SArtur Paszkiewicz if (!ctx) {
215e49ccfc8SArtur Paszkiewicz rc = -ENOMEM;
216e49ccfc8SArtur Paszkiewicz goto error;
217e49ccfc8SArtur Paszkiewicz }
218e49ccfc8SArtur Paszkiewicz ctx->cb_fn = cb_fn;
219e49ccfc8SArtur Paszkiewicz ctx->cb_arg = cb_arg;
220e49ccfc8SArtur Paszkiewicz
221e49ccfc8SArtur Paszkiewicz rc = ftl_mngt_call_dev_shutdown(dev, dev_free_cb, ctx);
222e49ccfc8SArtur Paszkiewicz if (rc) {
223e49ccfc8SArtur Paszkiewicz goto error;
224e49ccfc8SArtur Paszkiewicz }
225e49ccfc8SArtur Paszkiewicz
226e49ccfc8SArtur Paszkiewicz return 0;
227e49ccfc8SArtur Paszkiewicz
228e49ccfc8SArtur Paszkiewicz error:
229e49ccfc8SArtur Paszkiewicz free(ctx);
230e49ccfc8SArtur Paszkiewicz return rc;
231e49ccfc8SArtur Paszkiewicz }
232e49ccfc8SArtur Paszkiewicz
233e49ccfc8SArtur Paszkiewicz SPDK_LOG_REGISTER_COMPONENT(ftl_init)
234