1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018-2022 Advanced Micro Devices, Inc.
3 */
4
5 #include <stdbool.h>
6
7 #include <rte_memzone.h>
8
9 #include "ionic.h"
10 #include "ionic_ethdev.h"
11 #include "ionic_lif.h"
12
13 static const char *
ionic_error_to_str(enum ionic_status_code code)14 ionic_error_to_str(enum ionic_status_code code)
15 {
16 switch (code) {
17 case IONIC_RC_SUCCESS:
18 return "IONIC_RC_SUCCESS";
19 case IONIC_RC_EVERSION:
20 return "IONIC_RC_EVERSION";
21 case IONIC_RC_EOPCODE:
22 return "IONIC_RC_EOPCODE";
23 case IONIC_RC_EIO:
24 return "IONIC_RC_EIO";
25 case IONIC_RC_EPERM:
26 return "IONIC_RC_EPERM";
27 case IONIC_RC_EQID:
28 return "IONIC_RC_EQID";
29 case IONIC_RC_EQTYPE:
30 return "IONIC_RC_EQTYPE";
31 case IONIC_RC_ENOENT:
32 return "IONIC_RC_ENOENT";
33 case IONIC_RC_EINTR:
34 return "IONIC_RC_EINTR";
35 case IONIC_RC_EAGAIN:
36 return "IONIC_RC_EAGAIN";
37 case IONIC_RC_ENOMEM:
38 return "IONIC_RC_ENOMEM";
39 case IONIC_RC_EFAULT:
40 return "IONIC_RC_EFAULT";
41 case IONIC_RC_EBUSY:
42 return "IONIC_RC_EBUSY";
43 case IONIC_RC_EEXIST:
44 return "IONIC_RC_EEXIST";
45 case IONIC_RC_EINVAL:
46 return "IONIC_RC_EINVAL";
47 case IONIC_RC_ENOSPC:
48 return "IONIC_RC_ENOSPC";
49 case IONIC_RC_ERANGE:
50 return "IONIC_RC_ERANGE";
51 case IONIC_RC_BAD_ADDR:
52 return "IONIC_RC_BAD_ADDR";
53 case IONIC_RC_DEV_CMD:
54 return "IONIC_RC_DEV_CMD";
55 case IONIC_RC_ERROR:
56 return "IONIC_RC_ERROR";
57 case IONIC_RC_ERDMA:
58 return "IONIC_RC_ERDMA";
59 default:
60 return "IONIC_RC_UNKNOWN";
61 }
62 }
63
64 const char *
ionic_opcode_to_str(enum ionic_cmd_opcode opcode)65 ionic_opcode_to_str(enum ionic_cmd_opcode opcode)
66 {
67 switch (opcode) {
68 case IONIC_CMD_NOP:
69 return "IONIC_CMD_NOP";
70 case IONIC_CMD_INIT:
71 return "IONIC_CMD_INIT";
72 case IONIC_CMD_RESET:
73 return "IONIC_CMD_RESET";
74 case IONIC_CMD_IDENTIFY:
75 return "IONIC_CMD_IDENTIFY";
76 case IONIC_CMD_GETATTR:
77 return "IONIC_CMD_GETATTR";
78 case IONIC_CMD_SETATTR:
79 return "IONIC_CMD_SETATTR";
80 case IONIC_CMD_PORT_IDENTIFY:
81 return "IONIC_CMD_PORT_IDENTIFY";
82 case IONIC_CMD_PORT_INIT:
83 return "IONIC_CMD_PORT_INIT";
84 case IONIC_CMD_PORT_RESET:
85 return "IONIC_CMD_PORT_RESET";
86 case IONIC_CMD_PORT_GETATTR:
87 return "IONIC_CMD_PORT_GETATTR";
88 case IONIC_CMD_PORT_SETATTR:
89 return "IONIC_CMD_PORT_SETATTR";
90 case IONIC_CMD_LIF_INIT:
91 return "IONIC_CMD_LIF_INIT";
92 case IONIC_CMD_LIF_RESET:
93 return "IONIC_CMD_LIF_RESET";
94 case IONIC_CMD_LIF_IDENTIFY:
95 return "IONIC_CMD_LIF_IDENTIFY";
96 case IONIC_CMD_LIF_SETATTR:
97 return "IONIC_CMD_LIF_SETATTR";
98 case IONIC_CMD_LIF_GETATTR:
99 return "IONIC_CMD_LIF_GETATTR";
100 case IONIC_CMD_RX_MODE_SET:
101 return "IONIC_CMD_RX_MODE_SET";
102 case IONIC_CMD_RX_FILTER_ADD:
103 return "IONIC_CMD_RX_FILTER_ADD";
104 case IONIC_CMD_RX_FILTER_DEL:
105 return "IONIC_CMD_RX_FILTER_DEL";
106 case IONIC_CMD_Q_INIT:
107 return "IONIC_CMD_Q_INIT";
108 case IONIC_CMD_Q_CONTROL:
109 return "IONIC_CMD_Q_CONTROL";
110 case IONIC_CMD_Q_IDENTIFY:
111 return "IONIC_CMD_Q_IDENTIFY";
112 case IONIC_CMD_RDMA_RESET_LIF:
113 return "IONIC_CMD_RDMA_RESET_LIF";
114 case IONIC_CMD_RDMA_CREATE_EQ:
115 return "IONIC_CMD_RDMA_CREATE_EQ";
116 case IONIC_CMD_RDMA_CREATE_CQ:
117 return "IONIC_CMD_RDMA_CREATE_CQ";
118 case IONIC_CMD_RDMA_CREATE_ADMINQ:
119 return "IONIC_CMD_RDMA_CREATE_ADMINQ";
120 default:
121 return "DEVCMD_UNKNOWN";
122 }
123 }
124
125 static int
ionic_adminq_check_err(struct ionic_admin_ctx * ctx,bool timeout)126 ionic_adminq_check_err(struct ionic_admin_ctx *ctx, bool timeout)
127 {
128 const char *name;
129 const char *status;
130
131 name = ionic_opcode_to_str(ctx->cmd.cmd.opcode);
132
133 if (ctx->comp.comp.status || timeout) {
134 status = ionic_error_to_str(ctx->comp.comp.status);
135 IONIC_PRINT(ERR, "%s (%d) failed: %s (%d)",
136 name,
137 ctx->cmd.cmd.opcode,
138 timeout ? "TIMEOUT" : status,
139 timeout ? -1 : ctx->comp.comp.status);
140 return -EIO;
141 }
142
143 IONIC_PRINT(DEBUG, "%s (%d) succeeded", name, ctx->cmd.cmd.opcode);
144
145 return 0;
146 }
147
148 static bool
ionic_adminq_service(struct ionic_cq * cq,uint16_t cq_desc_index,void * cb_arg __rte_unused)149 ionic_adminq_service(struct ionic_cq *cq, uint16_t cq_desc_index,
150 void *cb_arg __rte_unused)
151 {
152 struct ionic_admin_comp *cq_desc_base = cq->base;
153 struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
154 struct ionic_qcq *qcq = IONIC_CQ_TO_QCQ(cq);
155 struct ionic_queue *q = &qcq->q;
156 struct ionic_admin_ctx *ctx;
157 uint16_t curr_q_tail_idx;
158 uint16_t stop_index;
159 void **info;
160
161 if (!color_match(cq_desc->color, cq->done_color))
162 return false;
163
164 stop_index = rte_le_to_cpu_16(cq_desc->comp_index);
165
166 do {
167 info = IONIC_INFO_PTR(q, q->tail_idx);
168
169 ctx = info[0];
170 if (ctx) {
171 memcpy(&ctx->comp, cq_desc, sizeof(*cq_desc));
172
173 ctx->pending_work = false; /* done */
174 }
175
176 curr_q_tail_idx = q->tail_idx;
177 q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
178 } while (curr_q_tail_idx != stop_index);
179
180 return true;
181 }
182
183 uint16_t
ionic_adminq_space_avail(struct ionic_lif * lif)184 ionic_adminq_space_avail(struct ionic_lif *lif)
185 {
186 return ionic_q_space_avail(&lif->adminqcq->qcq.q);
187 }
188
189 /** ionic_adminq_post - Post an admin command.
190 * @lif: Handle to lif.
191 * @cmd_ctx: Api admin command context.
192 *
193 * Post the command to an admin queue in the ethernet driver. If this command
194 * succeeds, then the command has been posted, but that does not indicate a
195 * completion. If this command returns success, then the completion callback
196 * will eventually be called.
197 *
198 * Return: zero or negative error status.
199 */
200 int
ionic_adminq_post(struct ionic_lif * lif,struct ionic_admin_ctx * ctx)201 ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx)
202 {
203 struct ionic_queue *q = &lif->adminqcq->qcq.q;
204 struct ionic_admin_cmd *q_desc_base = q->base;
205 struct ionic_admin_cmd *q_desc;
206 void **info;
207 int err = 0;
208
209 rte_spinlock_lock(&lif->adminq_lock);
210
211 if (ionic_q_space_avail(q) < 1) {
212 err = -ENOSPC;
213 goto err_out;
214 }
215
216 q_desc = &q_desc_base[q->head_idx];
217
218 memcpy(q_desc, &ctx->cmd, sizeof(ctx->cmd));
219
220 info = IONIC_INFO_PTR(q, q->head_idx);
221 info[0] = ctx;
222
223 q->head_idx = Q_NEXT_TO_POST(q, 1);
224
225 /* Ring doorbell */
226 ionic_q_flush(q);
227
228 err_out:
229 rte_spinlock_unlock(&lif->adminq_lock);
230
231 return err;
232 }
233
234 static int
ionic_adminq_wait_for_completion(struct ionic_lif * lif,struct ionic_admin_ctx * ctx,unsigned long max_wait)235 ionic_adminq_wait_for_completion(struct ionic_lif *lif,
236 struct ionic_admin_ctx *ctx, unsigned long max_wait)
237 {
238 struct ionic_queue *q = &lif->adminqcq->qcq.q;
239 unsigned long step_usec = IONIC_DEVCMD_CHECK_PERIOD_US;
240 unsigned long step_deadline;
241 unsigned long max_wait_usec = max_wait * 1000000L;
242 unsigned long elapsed_usec = 0;
243 int budget = 8;
244 uint16_t idx;
245 void **info;
246
247 step_deadline = IONIC_ADMINQ_WDOG_MS * 1000 / step_usec;
248
249 while (ctx->pending_work && elapsed_usec < max_wait_usec) {
250 /*
251 * Locking here as adminq is served inline and could be
252 * called from multiple places
253 */
254 rte_spinlock_lock(&lif->adminq_service_lock);
255
256 ionic_qcq_service(&lif->adminqcq->qcq, budget,
257 ionic_adminq_service, NULL);
258
259 /*
260 * Ring the doorbell again if work is pending after deadline.
261 */
262 if (ctx->pending_work && !step_deadline) {
263 step_deadline = IONIC_ADMINQ_WDOG_MS *
264 1000 / step_usec;
265
266 rte_spinlock_lock(&lif->adminq_lock);
267 idx = Q_NEXT_TO_POST(q, -1);
268 info = IONIC_INFO_PTR(q, idx);
269 if (info[0] == ctx)
270 ionic_q_flush(q);
271 rte_spinlock_unlock(&lif->adminq_lock);
272 }
273
274 rte_spinlock_unlock(&lif->adminq_service_lock);
275
276 rte_delay_us_block(step_usec);
277 elapsed_usec += step_usec;
278 step_deadline--;
279 }
280
281 return (!ctx->pending_work);
282 }
283
284 int
ionic_adminq_post_wait(struct ionic_lif * lif,struct ionic_admin_ctx * ctx)285 ionic_adminq_post_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx)
286 {
287 int err;
288
289 IONIC_PRINT(DEBUG, "Sending %s (%d) via the admin queue",
290 ionic_opcode_to_str(ctx->cmd.cmd.opcode), ctx->cmd.cmd.opcode);
291
292 err = ionic_adminq_post(lif, ctx);
293 if (err) {
294 IONIC_PRINT(ERR, "Failure posting %d to the admin queue (%d)",
295 ctx->cmd.cmd.opcode, err);
296 return err;
297 }
298
299 return ionic_adminq_wait(lif, ctx);
300 }
301
302 int
ionic_adminq_wait(struct ionic_lif * lif,struct ionic_admin_ctx * ctx)303 ionic_adminq_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx)
304 {
305 bool done;
306
307 done = ionic_adminq_wait_for_completion(lif, ctx,
308 IONIC_DEVCMD_TIMEOUT);
309
310 return ionic_adminq_check_err(ctx, !done /* timed out */);
311 }
312
313 static int
ionic_dev_cmd_wait(struct ionic_dev * idev,unsigned long max_wait)314 ionic_dev_cmd_wait(struct ionic_dev *idev, unsigned long max_wait)
315 {
316 unsigned long step_usec = IONIC_DEVCMD_CHECK_PERIOD_US;
317 unsigned long max_wait_usec = max_wait * 1000000L;
318 unsigned long elapsed_usec = 0;
319 int done;
320
321 /* Wait for dev cmd to complete.. but no more than max_wait sec */
322
323 do {
324 done = ionic_dev_cmd_done(idev);
325 if (done) {
326 IONIC_PRINT(DEBUG, "DEVCMD %d done took %ld usecs",
327 ioread8(&idev->dev_cmd->cmd.cmd.opcode),
328 elapsed_usec);
329 return 0;
330 }
331
332 rte_delay_us_block(step_usec);
333
334 elapsed_usec += step_usec;
335 } while (elapsed_usec < max_wait_usec);
336
337 IONIC_PRINT(ERR, "DEVCMD %d timeout after %ld usecs",
338 ioread8(&idev->dev_cmd->cmd.cmd.opcode),
339 elapsed_usec);
340
341 return -ETIMEDOUT;
342 }
343
344 static int
ionic_dev_cmd_check_error(struct ionic_dev * idev)345 ionic_dev_cmd_check_error(struct ionic_dev *idev)
346 {
347 uint8_t status;
348
349 status = ionic_dev_cmd_status(idev);
350 if (status == IONIC_RC_SUCCESS)
351 return 0;
352
353 return (status == IONIC_RC_EAGAIN) ? -EAGAIN : -EIO;
354 }
355
356 int
ionic_dev_cmd_wait_check(struct ionic_dev * idev,unsigned long max_wait)357 ionic_dev_cmd_wait_check(struct ionic_dev *idev, unsigned long max_wait)
358 {
359 int err;
360
361 err = ionic_dev_cmd_wait(idev, max_wait);
362
363 if (!err)
364 err = ionic_dev_cmd_check_error(idev);
365
366 IONIC_PRINT(DEBUG, "dev_cmd returned %d", err);
367 return err;
368 }
369
370 int
ionic_setup(struct ionic_adapter * adapter)371 ionic_setup(struct ionic_adapter *adapter)
372 {
373 return (*adapter->intf->setup)(adapter);
374 }
375
376 int
ionic_identify(struct ionic_adapter * adapter)377 ionic_identify(struct ionic_adapter *adapter)
378 {
379 struct ionic_dev *idev = &adapter->idev;
380 struct ionic_identity *ident = &adapter->ident;
381 uint32_t drv_size = RTE_DIM(ident->drv.words);
382 uint32_t cmd_size = RTE_DIM(idev->dev_cmd->data);
383 uint32_t dev_size = RTE_DIM(ident->dev.words);
384 uint32_t i, nwords;
385 int err;
386
387 memset(ident, 0, sizeof(*ident));
388
389 ident->drv.os_type = IONIC_OS_TYPE_LINUX;
390 ident->drv.os_dist = 0;
391 snprintf(ident->drv.os_dist_str,
392 sizeof(ident->drv.os_dist_str), "Unknown");
393 ident->drv.kernel_ver = 0;
394 snprintf(ident->drv.kernel_ver_str,
395 sizeof(ident->drv.kernel_ver_str), "DPDK");
396 strncpy(ident->drv.driver_ver_str, IONIC_DRV_VERSION,
397 sizeof(ident->drv.driver_ver_str) - 1);
398
399 nwords = RTE_MIN(drv_size, cmd_size);
400 for (i = 0; i < nwords; i++)
401 iowrite32(ident->drv.words[i], &idev->dev_cmd->data[i]);
402
403 ionic_dev_cmd_identify(idev, IONIC_IDENTITY_VERSION_1);
404 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
405 if (!err) {
406 nwords = RTE_MIN(dev_size, cmd_size);
407 for (i = 0; i < nwords; i++)
408 ident->dev.words[i] = ioread32(&idev->dev_cmd->data[i]);
409 }
410
411 return err;
412 }
413
414 int
ionic_init(struct ionic_adapter * adapter)415 ionic_init(struct ionic_adapter *adapter)
416 {
417 struct ionic_dev *idev = &adapter->idev;
418
419 ionic_dev_cmd_init(idev);
420 return ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
421 }
422
423 int
ionic_reset(struct ionic_adapter * adapter)424 ionic_reset(struct ionic_adapter *adapter)
425 {
426 struct ionic_dev *idev = &adapter->idev;
427
428 ionic_dev_cmd_reset(idev);
429 return ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
430 }
431
432 int
ionic_port_identify(struct ionic_adapter * adapter)433 ionic_port_identify(struct ionic_adapter *adapter)
434 {
435 struct ionic_dev *idev = &adapter->idev;
436 struct ionic_identity *ident = &adapter->ident;
437 uint32_t port_words = RTE_DIM(ident->port.words);
438 uint32_t cmd_words = RTE_DIM(idev->dev_cmd->data);
439 uint32_t i, nwords;
440 int err;
441
442 ionic_dev_cmd_port_identify(idev);
443 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
444 if (!err) {
445 nwords = RTE_MIN(port_words, cmd_words);
446 for (i = 0; i < nwords; i++)
447 ident->port.words[i] =
448 ioread32(&idev->dev_cmd->data[i]);
449 }
450
451 IONIC_PRINT(INFO, "speed %d",
452 rte_le_to_cpu_32(ident->port.config.speed));
453 IONIC_PRINT(INFO, "mtu %d",
454 rte_le_to_cpu_32(ident->port.config.mtu));
455 IONIC_PRINT(INFO, "state %d", ident->port.config.state);
456 IONIC_PRINT(INFO, "an_enable %d", ident->port.config.an_enable);
457 IONIC_PRINT(INFO, "fec_type %d", ident->port.config.fec_type);
458 IONIC_PRINT(INFO, "pause_type %d", ident->port.config.pause_type);
459 IONIC_PRINT(INFO, "loopback_mode %d",
460 ident->port.config.loopback_mode);
461
462 return err;
463 }
464
465 static const struct rte_memzone *
ionic_memzone_reserve(const char * name,uint32_t len,int socket_id)466 ionic_memzone_reserve(const char *name, uint32_t len, int socket_id)
467 {
468 const struct rte_memzone *mz;
469
470 mz = rte_memzone_lookup(name);
471 if (mz)
472 return mz;
473
474 mz = rte_memzone_reserve_aligned(name, len, socket_id,
475 RTE_MEMZONE_IOVA_CONTIG, IONIC_ALIGN);
476 return mz;
477 }
478
479 int
ionic_port_init(struct ionic_adapter * adapter)480 ionic_port_init(struct ionic_adapter *adapter)
481 {
482 struct ionic_dev *idev = &adapter->idev;
483 struct ionic_identity *ident = &adapter->ident;
484 char z_name[RTE_MEMZONE_NAMESIZE];
485 uint32_t config_words = RTE_DIM(ident->port.config.words);
486 uint32_t cmd_words = RTE_DIM(idev->dev_cmd->data);
487 uint32_t i, nwords;
488 int err;
489
490 if (idev->port_info)
491 return 0;
492
493 idev->port_info_sz = RTE_ALIGN(sizeof(*idev->port_info),
494 rte_mem_page_size());
495
496 snprintf(z_name, sizeof(z_name), "%s_port_%s_info",
497 IONIC_DRV_NAME, adapter->name);
498
499 idev->port_info_z = ionic_memzone_reserve(z_name, idev->port_info_sz,
500 SOCKET_ID_ANY);
501 if (!idev->port_info_z) {
502 IONIC_PRINT(ERR, "Cannot reserve port info DMA memory");
503 return -ENOMEM;
504 }
505
506 idev->port_info = idev->port_info_z->addr;
507 idev->port_info_pa = idev->port_info_z->iova;
508
509 nwords = RTE_MIN(config_words, cmd_words);
510
511 for (i = 0; i < nwords; i++)
512 iowrite32(ident->port.config.words[i], &idev->dev_cmd->data[i]);
513
514 idev->port_info->config.state = IONIC_PORT_ADMIN_STATE_UP;
515 ionic_dev_cmd_port_init(idev);
516 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
517 if (err)
518 IONIC_PRINT(ERR, "Failed to init port");
519
520 return err;
521 }
522
523 int
ionic_port_reset(struct ionic_adapter * adapter)524 ionic_port_reset(struct ionic_adapter *adapter)
525 {
526 struct ionic_dev *idev = &adapter->idev;
527 int err;
528
529 if (!idev->port_info)
530 return 0;
531
532 ionic_dev_cmd_port_reset(idev);
533 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
534 if (err) {
535 IONIC_PRINT(ERR, "Failed to reset port");
536 return err;
537 }
538
539 idev->port_info = NULL;
540 idev->port_info_pa = 0;
541
542 return 0;
543 }
544