1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2019 Mellanox Technologies, Ltd 3 */ 4 #include <netinet/in.h> 5 6 #include <rte_malloc.h> 7 #include <rte_errno.h> 8 #include <rte_common.h> 9 10 #include <mlx5_common.h> 11 12 #include "mlx5_vdpa_utils.h" 13 #include "mlx5_vdpa.h" 14 15 static void 16 mlx5_vdpa_rss_flows_destroy(struct mlx5_vdpa_priv *priv) 17 { 18 unsigned i; 19 20 for (i = 0; i < RTE_DIM(priv->steer.rss); ++i) { 21 if (priv->steer.rss[i].flow) { 22 claim_zero(mlx5_glue->dv_destroy_flow 23 (priv->steer.rss[i].flow)); 24 priv->steer.rss[i].flow = NULL; 25 } 26 if (priv->steer.rss[i].tir_action) { 27 claim_zero(mlx5_glue->destroy_flow_action 28 (priv->steer.rss[i].tir_action)); 29 priv->steer.rss[i].tir_action = NULL; 30 } 31 if (priv->steer.rss[i].tir) { 32 claim_zero(mlx5_devx_cmd_destroy 33 (priv->steer.rss[i].tir)); 34 priv->steer.rss[i].tir = NULL; 35 } 36 if (priv->steer.rss[i].matcher) { 37 claim_zero(mlx5_glue->dv_destroy_flow_matcher 38 (priv->steer.rss[i].matcher)); 39 priv->steer.rss[i].matcher = NULL; 40 } 41 } 42 } 43 44 void 45 mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv *priv) 46 { 47 mlx5_vdpa_rss_flows_destroy(priv); 48 if (priv->steer.tbl) { 49 claim_zero(mlx5_glue->dr_destroy_flow_tbl(priv->steer.tbl)); 50 priv->steer.tbl = NULL; 51 } 52 if (priv->steer.domain) { 53 claim_zero(mlx5_glue->dr_destroy_domain(priv->steer.domain)); 54 priv->steer.domain = NULL; 55 } 56 if (priv->steer.rqt) { 57 claim_zero(mlx5_devx_cmd_destroy(priv->steer.rqt)); 58 priv->steer.rqt = NULL; 59 } 60 } 61 62 #define MLX5_VDPA_DEFAULT_RQT_SIZE 512 63 /* 64 * Return the number of queues configured to the table on success, otherwise 65 * -1 on error. 66 */ 67 static int 68 mlx5_vdpa_rqt_prepare(struct mlx5_vdpa_priv *priv) 69 { 70 int i; 71 uint32_t rqt_n = RTE_MIN(MLX5_VDPA_DEFAULT_RQT_SIZE, 72 1 << priv->log_max_rqt_size); 73 struct mlx5_devx_rqt_attr *attr = rte_zmalloc(__func__, sizeof(*attr) 74 + rqt_n * 75 sizeof(uint32_t), 0); 76 uint32_t k = 0, j; 77 int ret = 0, num; 78 79 if (!attr) { 80 DRV_LOG(ERR, "Failed to allocate RQT attributes memory."); 81 rte_errno = ENOMEM; 82 return -ENOMEM; 83 } 84 for (i = 0; i < priv->nr_virtqs; i++) { 85 if (is_virtq_recvq(i, priv->nr_virtqs) && 86 priv->virtqs[i].enable && priv->virtqs[i].virtq) { 87 attr->rq_list[k] = priv->virtqs[i].virtq->id; 88 k++; 89 } 90 } 91 if (k == 0) 92 /* No enabled RQ to configure for RSS. */ 93 return 0; 94 num = (int)k; 95 for (j = 0; k != rqt_n; ++k, ++j) 96 attr->rq_list[k] = attr->rq_list[j]; 97 attr->rq_type = MLX5_INLINE_Q_TYPE_VIRTQ; 98 attr->rqt_max_size = rqt_n; 99 attr->rqt_actual_size = rqt_n; 100 if (!priv->steer.rqt) { 101 priv->steer.rqt = mlx5_devx_cmd_create_rqt(priv->ctx, attr); 102 if (!priv->steer.rqt) { 103 DRV_LOG(ERR, "Failed to create RQT."); 104 ret = -rte_errno; 105 } 106 } else { 107 ret = mlx5_devx_cmd_modify_rqt(priv->steer.rqt, attr); 108 if (ret) 109 DRV_LOG(ERR, "Failed to modify RQT."); 110 } 111 rte_free(attr); 112 return ret ? -1 : num; 113 } 114 115 static int __rte_unused 116 mlx5_vdpa_rss_flows_create(struct mlx5_vdpa_priv *priv) 117 { 118 #ifdef HAVE_MLX5DV_DR 119 struct mlx5_devx_tir_attr tir_att = { 120 .disp_type = MLX5_TIRC_DISP_TYPE_INDIRECT, 121 .rx_hash_fn = MLX5_RX_HASH_FN_TOEPLITZ, 122 .transport_domain = priv->td->id, 123 .indirect_table = priv->steer.rqt->id, 124 .rx_hash_symmetric = 1, 125 .rx_hash_toeplitz_key = { 0x2c, 0xc6, 0x81, 0xd1, 126 0x5b, 0xdb, 0xf4, 0xf7, 127 0xfc, 0xa2, 0x83, 0x19, 128 0xdb, 0x1a, 0x3e, 0x94, 129 0x6b, 0x9e, 0x38, 0xd9, 130 0x2c, 0x9c, 0x03, 0xd1, 131 0xad, 0x99, 0x44, 0xa7, 132 0xd9, 0x56, 0x3d, 0x59, 133 0x06, 0x3c, 0x25, 0xf3, 134 0xfc, 0x1f, 0xdc, 0x2a }, 135 }; 136 struct { 137 size_t size; 138 /**< Size of match value. Do NOT split size and key! */ 139 uint32_t buf[MLX5_ST_SZ_DW(fte_match_param)]; 140 /**< Matcher value. This value is used as the mask or a key. */ 141 } matcher_mask = { 142 .size = sizeof(matcher_mask.buf) - 143 MLX5_ST_SZ_BYTES(fte_match_set_misc4), 144 }, 145 matcher_value = { 146 .size = sizeof(matcher_value.buf) - 147 MLX5_ST_SZ_BYTES(fte_match_set_misc4), 148 }; 149 struct mlx5dv_flow_matcher_attr dv_attr = { 150 .type = IBV_FLOW_ATTR_NORMAL, 151 .match_mask = (void *)&matcher_mask, 152 }; 153 void *match_m = matcher_mask.buf; 154 void *match_v = matcher_value.buf; 155 void *headers_m = MLX5_ADDR_OF(fte_match_param, match_m, outer_headers); 156 void *headers_v = MLX5_ADDR_OF(fte_match_param, match_v, outer_headers); 157 void *actions[1]; 158 const uint8_t l3_hash = 159 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP) | 160 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP); 161 const uint8_t l4_hash = 162 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT) | 163 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT); 164 enum { PRIO, CRITERIA, IP_VER_M, IP_VER_V, IP_PROT_M, IP_PROT_V, L3_BIT, 165 L4_BIT, HASH, END}; 166 const uint8_t vars[RTE_DIM(priv->steer.rss)][END] = { 167 { 7, 0, 0, 0, 0, 0, 0, 0, 0 }, 168 { 6, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0, 0, 169 MLX5_L3_PROT_TYPE_IPV4, 0, l3_hash }, 170 { 6, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0, 0, 171 MLX5_L3_PROT_TYPE_IPV6, 0, l3_hash }, 172 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0xff, 173 IPPROTO_UDP, MLX5_L3_PROT_TYPE_IPV4, MLX5_L4_PROT_TYPE_UDP, 174 l3_hash | l4_hash }, 175 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0xff, 176 IPPROTO_TCP, MLX5_L3_PROT_TYPE_IPV4, MLX5_L4_PROT_TYPE_TCP, 177 l3_hash | l4_hash }, 178 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0xff, 179 IPPROTO_UDP, MLX5_L3_PROT_TYPE_IPV6, MLX5_L4_PROT_TYPE_UDP, 180 l3_hash | l4_hash }, 181 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0xff, 182 IPPROTO_TCP, MLX5_L3_PROT_TYPE_IPV6, MLX5_L4_PROT_TYPE_TCP, 183 l3_hash | l4_hash }, 184 }; 185 unsigned i; 186 187 for (i = 0; i < RTE_DIM(priv->steer.rss); ++i) { 188 dv_attr.priority = vars[i][PRIO]; 189 dv_attr.match_criteria_enable = vars[i][CRITERIA]; 190 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 191 vars[i][IP_VER_M]); 192 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 193 vars[i][IP_VER_V]); 194 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol, 195 vars[i][IP_PROT_M]); 196 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 197 vars[i][IP_PROT_V]); 198 tir_att.rx_hash_field_selector_outer.l3_prot_type = 199 vars[i][L3_BIT]; 200 tir_att.rx_hash_field_selector_outer.l4_prot_type = 201 vars[i][L4_BIT]; 202 tir_att.rx_hash_field_selector_outer.selected_fields = 203 vars[i][HASH]; 204 priv->steer.rss[i].matcher = mlx5_glue->dv_create_flow_matcher 205 (priv->ctx, &dv_attr, priv->steer.tbl); 206 if (!priv->steer.rss[i].matcher) { 207 DRV_LOG(ERR, "Failed to create matcher %d.", i); 208 goto error; 209 } 210 priv->steer.rss[i].tir = mlx5_devx_cmd_create_tir(priv->ctx, 211 &tir_att); 212 if (!priv->steer.rss[i].tir) { 213 DRV_LOG(ERR, "Failed to create TIR %d.", i); 214 goto error; 215 } 216 priv->steer.rss[i].tir_action = 217 mlx5_glue->dv_create_flow_action_dest_devx_tir 218 (priv->steer.rss[i].tir->obj); 219 if (!priv->steer.rss[i].tir_action) { 220 DRV_LOG(ERR, "Failed to create TIR action %d.", i); 221 goto error; 222 } 223 actions[0] = priv->steer.rss[i].tir_action; 224 priv->steer.rss[i].flow = mlx5_glue->dv_create_flow 225 (priv->steer.rss[i].matcher, 226 (void *)&matcher_value, 1, actions); 227 if (!priv->steer.rss[i].flow) { 228 DRV_LOG(ERR, "Failed to create flow %d.", i); 229 goto error; 230 } 231 } 232 return 0; 233 error: 234 /* Resources will be freed by the caller. */ 235 return -1; 236 #else 237 (void)priv; 238 return -ENOTSUP; 239 #endif /* HAVE_MLX5DV_DR */ 240 } 241 242 int 243 mlx5_vdpa_steer_update(struct mlx5_vdpa_priv *priv) 244 { 245 int ret = mlx5_vdpa_rqt_prepare(priv); 246 247 if (ret == 0) { 248 mlx5_vdpa_rss_flows_destroy(priv); 249 if (priv->steer.rqt) { 250 claim_zero(mlx5_devx_cmd_destroy(priv->steer.rqt)); 251 priv->steer.rqt = NULL; 252 } 253 } else if (ret < 0) { 254 return ret; 255 } else if (!priv->steer.rss[0].flow) { 256 ret = mlx5_vdpa_rss_flows_create(priv); 257 if (ret) { 258 DRV_LOG(ERR, "Cannot create RSS flows."); 259 return -1; 260 } 261 } 262 return 0; 263 } 264 265 int 266 mlx5_vdpa_steer_setup(struct mlx5_vdpa_priv *priv) 267 { 268 #ifdef HAVE_MLX5DV_DR 269 priv->steer.domain = mlx5_glue->dr_create_domain(priv->ctx, 270 MLX5DV_DR_DOMAIN_TYPE_NIC_RX); 271 if (!priv->steer.domain) { 272 DRV_LOG(ERR, "Failed to create Rx domain."); 273 goto error; 274 } 275 priv->steer.tbl = mlx5_glue->dr_create_flow_tbl(priv->steer.domain, 0); 276 if (!priv->steer.tbl) { 277 DRV_LOG(ERR, "Failed to create table 0 with Rx domain."); 278 goto error; 279 } 280 if (mlx5_vdpa_steer_update(priv)) 281 goto error; 282 return 0; 283 error: 284 mlx5_vdpa_steer_unset(priv); 285 return -1; 286 #else 287 (void)priv; 288 return -ENOTSUP; 289 #endif /* HAVE_MLX5DV_DR */ 290 } 291