1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017 6WIND S.A.
3 * Copyright 2017 Mellanox Technologies, Ltd
4 */
5
6 /**
7 * @file
8 * Interrupts handling for mlx4 driver.
9 */
10
11 #include <errno.h>
12 #include <stdint.h>
13 #include <stdlib.h>
14
15 /* Verbs headers do not support -pedantic. */
16 #ifdef PEDANTIC
17 #pragma GCC diagnostic ignored "-Wpedantic"
18 #endif
19 #include <infiniband/verbs.h>
20 #ifdef PEDANTIC
21 #pragma GCC diagnostic error "-Wpedantic"
22 #endif
23
24 #include <rte_alarm.h>
25 #include <rte_errno.h>
26 #include <ethdev_driver.h>
27 #include <rte_io.h>
28 #include <rte_interrupts.h>
29
30 #include "mlx4.h"
31 #include "mlx4_glue.h"
32 #include "mlx4_rxtx.h"
33 #include "mlx4_utils.h"
34
35 static int mlx4_link_status_check(struct mlx4_priv *priv);
36
37 /**
38 * Clean up Rx interrupts handler.
39 *
40 * @param priv
41 * Pointer to private structure.
42 */
43 static void
mlx4_rx_intr_vec_disable(struct mlx4_priv * priv)44 mlx4_rx_intr_vec_disable(struct mlx4_priv *priv)
45 {
46 struct rte_intr_handle *intr_handle = priv->intr_handle;
47
48 rte_intr_free_epoll_fd(intr_handle);
49 rte_intr_vec_list_free(intr_handle);
50
51 rte_intr_nb_efd_set(intr_handle, 0);
52 }
53
54 /**
55 * Allocate queue vector and fill epoll fd list for Rx interrupts.
56 *
57 * @param priv
58 * Pointer to private structure.
59 *
60 * @return
61 * 0 on success, negative errno value otherwise and rte_errno is set.
62 */
63 static int
mlx4_rx_intr_vec_enable(struct mlx4_priv * priv)64 mlx4_rx_intr_vec_enable(struct mlx4_priv *priv)
65 {
66 unsigned int i;
67 unsigned int rxqs_n = ETH_DEV(priv)->data->nb_rx_queues;
68 unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
69 unsigned int count = 0;
70 struct rte_intr_handle *intr_handle = priv->intr_handle;
71
72 mlx4_rx_intr_vec_disable(priv);
73 if (rte_intr_vec_list_alloc(intr_handle, NULL, n)) {
74 rte_errno = ENOMEM;
75 ERROR("failed to allocate memory for interrupt vector,"
76 " Rx interrupts will not be supported");
77 return -rte_errno;
78 }
79 for (i = 0; i != n; ++i) {
80 struct rxq *rxq = ETH_DEV(priv)->data->rx_queues[i];
81
82 /* Skip queues that cannot request interrupts. */
83 if (!rxq || !rxq->channel) {
84 /* Use invalid intr_vec[] index to disable entry. */
85 if (rte_intr_vec_list_index_set(intr_handle, i,
86 RTE_INTR_VEC_RXTX_OFFSET + RTE_MAX_RXTX_INTR_VEC_ID))
87 return -rte_errno;
88 continue;
89 }
90 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
91 rte_errno = E2BIG;
92 ERROR("too many Rx queues for interrupt vector size"
93 " (%d), Rx interrupts cannot be enabled",
94 RTE_MAX_RXTX_INTR_VEC_ID);
95 mlx4_rx_intr_vec_disable(priv);
96 return -rte_errno;
97 }
98
99 if (rte_intr_vec_list_index_set(intr_handle, i,
100 RTE_INTR_VEC_RXTX_OFFSET + count))
101 return -rte_errno;
102
103 if (rte_intr_efds_index_set(intr_handle, i,
104 rxq->channel->fd))
105 return -rte_errno;
106
107 count++;
108 }
109 if (!count)
110 mlx4_rx_intr_vec_disable(priv);
111 else if (rte_intr_nb_efd_set(intr_handle, count))
112 return -rte_errno;
113 return 0;
114 }
115
116 /**
117 * Process scheduled link status check.
118 *
119 * If LSC interrupts are requested, process related callback.
120 *
121 * @param priv
122 * Pointer to private structure.
123 */
124 static void
mlx4_link_status_alarm(struct mlx4_priv * priv)125 mlx4_link_status_alarm(struct mlx4_priv *priv)
126 {
127 const struct rte_eth_intr_conf *const intr_conf =
128 Ð_DEV(priv)->data->dev_conf.intr_conf;
129
130 MLX4_ASSERT(priv->intr_alarm == 1);
131 priv->intr_alarm = 0;
132 if (intr_conf->lsc && !mlx4_link_status_check(priv))
133 rte_eth_dev_callback_process(ETH_DEV(priv),
134 RTE_ETH_EVENT_INTR_LSC,
135 NULL);
136 }
137
138 /**
139 * Check link status.
140 *
141 * In case of inconsistency, another check is scheduled.
142 *
143 * @param priv
144 * Pointer to private structure.
145 *
146 * @return
147 * 0 on success (link status is consistent), negative errno value
148 * otherwise and rte_errno is set.
149 */
150 static int
mlx4_link_status_check(struct mlx4_priv * priv)151 mlx4_link_status_check(struct mlx4_priv *priv)
152 {
153 struct rte_eth_link *link = Ð_DEV(priv)->data->dev_link;
154 int ret = mlx4_link_update(ETH_DEV(priv), 0);
155
156 if (ret)
157 return ret;
158 if ((!link->link_speed && link->link_status) ||
159 (link->link_speed && !link->link_status)) {
160 if (!priv->intr_alarm) {
161 /* Inconsistent status, check again later. */
162 ret = rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
163 (void (*)(void *))
164 mlx4_link_status_alarm,
165 priv);
166 if (ret)
167 return ret;
168 priv->intr_alarm = 1;
169 }
170 rte_errno = EINPROGRESS;
171 return -rte_errno;
172 }
173 return 0;
174 }
175
176 /**
177 * Handle interrupts from the NIC.
178 *
179 * @param priv
180 * Pointer to private structure.
181 */
182 static void
mlx4_interrupt_handler(struct mlx4_priv * priv)183 mlx4_interrupt_handler(struct mlx4_priv *priv)
184 {
185 enum { LSC, RMV, };
186 static const enum rte_eth_event_type type[] = {
187 [LSC] = RTE_ETH_EVENT_INTR_LSC,
188 [RMV] = RTE_ETH_EVENT_INTR_RMV,
189 };
190 uint32_t caught[RTE_DIM(type)] = { 0 };
191 struct ibv_async_event event;
192 const struct rte_eth_intr_conf *const intr_conf =
193 Ð_DEV(priv)->data->dev_conf.intr_conf;
194 unsigned int i;
195
196 /* Read all message and acknowledge them. */
197 while (!mlx4_glue->get_async_event(priv->ctx, &event)) {
198 switch (event.event_type) {
199 case IBV_EVENT_PORT_ACTIVE:
200 case IBV_EVENT_PORT_ERR:
201 if (intr_conf->lsc && !mlx4_link_status_check(priv))
202 ++caught[LSC];
203 break;
204 case IBV_EVENT_DEVICE_FATAL:
205 if (intr_conf->rmv)
206 ++caught[RMV];
207 break;
208 default:
209 DEBUG("event type %d on physical port %d not handled",
210 event.event_type, event.element.port_num);
211 }
212 mlx4_glue->ack_async_event(&event);
213 }
214 for (i = 0; i != RTE_DIM(caught); ++i)
215 if (caught[i])
216 rte_eth_dev_callback_process(ETH_DEV(priv), type[i],
217 NULL);
218 }
219
220 /**
221 * MLX4 CQ notification .
222 *
223 * @param rxq
224 * Pointer to receive queue structure.
225 * @param solicited
226 * Is request solicited or not.
227 */
228 static void
mlx4_arm_cq(struct rxq * rxq,int solicited)229 mlx4_arm_cq(struct rxq *rxq, int solicited)
230 {
231 struct mlx4_cq *cq = &rxq->mcq;
232 uint64_t doorbell;
233 uint32_t sn = cq->arm_sn & MLX4_CQ_DB_GEQ_N_MASK;
234 uint32_t ci = cq->cons_index & MLX4_CQ_DB_CI_MASK;
235 uint32_t cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT;
236
237 *cq->arm_db = rte_cpu_to_be_32(sn << 28 | cmd | ci);
238 /*
239 * Make sure that the doorbell record in host memory is
240 * written before ringing the doorbell via PCI MMIO.
241 */
242 rte_wmb();
243 doorbell = sn << 28 | cmd | cq->cqn;
244 doorbell <<= 32;
245 doorbell |= ci;
246 rte_write64(rte_cpu_to_be_64(doorbell), cq->cq_db_reg);
247 }
248
249 /**
250 * Uninstall interrupt handler.
251 *
252 * @param priv
253 * Pointer to private structure.
254 *
255 * @return
256 * 0 on success, negative errno value otherwise and rte_errno is set.
257 */
258 int
mlx4_intr_uninstall(struct mlx4_priv * priv)259 mlx4_intr_uninstall(struct mlx4_priv *priv)
260 {
261 int err = rte_errno; /* Make sure rte_errno remains unchanged. */
262
263 if (rte_intr_fd_get(priv->intr_handle) != -1) {
264 rte_intr_callback_unregister(priv->intr_handle,
265 (void (*)(void *))
266 mlx4_interrupt_handler,
267 priv);
268 if (rte_intr_fd_set(priv->intr_handle, -1))
269 return -rte_errno;
270 }
271 rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv);
272 priv->intr_alarm = 0;
273 mlx4_rxq_intr_disable(priv);
274 rte_errno = err;
275 return 0;
276 }
277
278 /**
279 * Install interrupt handler.
280 *
281 * @param priv
282 * Pointer to private structure.
283 *
284 * @return
285 * 0 on success, negative errno value otherwise and rte_errno is set.
286 */
287 int
mlx4_intr_install(struct mlx4_priv * priv)288 mlx4_intr_install(struct mlx4_priv *priv)
289 {
290 const struct rte_eth_intr_conf *const intr_conf =
291 Ð_DEV(priv)->data->dev_conf.intr_conf;
292 int rc;
293
294 mlx4_intr_uninstall(priv);
295 if (intr_conf->lsc | intr_conf->rmv) {
296 if (rte_intr_fd_set(priv->intr_handle, priv->ctx->async_fd))
297 return -rte_errno;
298
299 rc = rte_intr_callback_register(priv->intr_handle,
300 (void (*)(void *))
301 mlx4_interrupt_handler,
302 priv);
303 if (rc < 0) {
304 rte_errno = -rc;
305 goto error;
306 }
307 }
308 return 0;
309 error:
310 mlx4_intr_uninstall(priv);
311 return -rte_errno;
312 }
313
314 /**
315 * DPDK callback for Rx queue interrupt disable.
316 *
317 * @param dev
318 * Pointer to Ethernet device structure.
319 * @param idx
320 * Rx queue index.
321 *
322 * @return
323 * 0 on success, negative errno value otherwise and rte_errno is set.
324 */
325 int
mlx4_rx_intr_disable(struct rte_eth_dev * dev,uint16_t idx)326 mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
327 {
328 struct rxq *rxq = dev->data->rx_queues[idx];
329 struct ibv_cq *ev_cq;
330 void *ev_ctx;
331 int ret;
332
333 if (!rxq || !rxq->channel) {
334 ret = EINVAL;
335 } else {
336 ret = mlx4_glue->get_cq_event(rxq->cq->channel, &ev_cq,
337 &ev_ctx);
338 /** For non-zero ret save the errno (may be EAGAIN
339 * which means the get_cq_event function was called before
340 * receiving one).
341 */
342 if (ret)
343 ret = errno;
344 else if (ev_cq != rxq->cq)
345 ret = EINVAL;
346 }
347 if (ret) {
348 rte_errno = ret;
349 if (ret != EAGAIN)
350 WARN("unable to disable interrupt on rx queue %d",
351 idx);
352 } else {
353 rxq->mcq.arm_sn++;
354 mlx4_glue->ack_cq_events(rxq->cq, 1);
355 }
356 return -ret;
357 }
358
359 /**
360 * DPDK callback for Rx queue interrupt enable.
361 *
362 * @param dev
363 * Pointer to Ethernet device structure.
364 * @param idx
365 * Rx queue index.
366 *
367 * @return
368 * 0 on success, negative errno value otherwise and rte_errno is set.
369 */
370 int
mlx4_rx_intr_enable(struct rte_eth_dev * dev,uint16_t idx)371 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
372 {
373 struct rxq *rxq = dev->data->rx_queues[idx];
374 int ret = 0;
375
376 if (!rxq || !rxq->channel) {
377 ret = EINVAL;
378 rte_errno = ret;
379 WARN("unable to arm interrupt on rx queue %d", idx);
380 } else {
381 mlx4_arm_cq(rxq, 0);
382 }
383 return -ret;
384 }
385
386 /**
387 * Enable datapath interrupts.
388 *
389 * @param priv
390 * Pointer to private structure.
391 *
392 * @return
393 * 0 on success, negative errno value otherwise and rte_errno is set.
394 */
395 int
mlx4_rxq_intr_enable(struct mlx4_priv * priv)396 mlx4_rxq_intr_enable(struct mlx4_priv *priv)
397 {
398 const struct rte_eth_intr_conf *const intr_conf =
399 Ð_DEV(priv)->data->dev_conf.intr_conf;
400
401 if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
402 goto error;
403 return 0;
404 error:
405 return -rte_errno;
406 }
407
408 /**
409 * Disable datapath interrupts, keeping other interrupts intact.
410 *
411 * @param priv
412 * Pointer to private structure.
413 */
414 void
mlx4_rxq_intr_disable(struct mlx4_priv * priv)415 mlx4_rxq_intr_disable(struct mlx4_priv *priv)
416 {
417 int err = rte_errno; /* Make sure rte_errno remains unchanged. */
418
419 mlx4_rx_intr_vec_disable(priv);
420 rte_errno = err;
421 }
422