1 /*- 2 * Copyright (c) 2020-2021, Mellanox Technologies, Ltd. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #include "opt_inet.h" 27 #include "opt_inet6.h" 28 29 #include <dev/mlx5/mlx5_en/en.h> 30 31 #include <dev/mlx5/mlx5_core/fs_core.h> 32 #include <dev/mlx5/mlx5_core/fs_tcp.h> 33 #include <dev/mlx5/device.h> 34 35 #include <sys/domain.h> 36 37 #include <netinet/in_pcb.h> 38 39 #if defined(INET) || defined(INET6) 40 static void 41 accel_fs_tcp_set_ipv4_flow(struct mlx5_flow_spec *spec, struct inpcb *inp) 42 { 43 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 44 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP); 45 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 46 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4); 47 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 48 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), 49 &inp->inp_faddr, 4); 50 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 51 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 52 &inp->inp_laddr, 4); 53 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 54 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); 55 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 56 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 57 } 58 #endif 59 60 #ifdef INET6 61 static void 62 accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec *spec, struct inpcb *inp) 63 { 64 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 65 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP); 66 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 67 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6); 68 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 69 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 70 &inp->in6p_faddr, 16); 71 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 72 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 73 &inp->in6p_laddr, 16); 74 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 75 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 76 0xff, 16); 77 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 78 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 79 0xff, 16); 80 } 81 #endif 82 83 void 84 mlx5e_accel_fs_del_inpcb(struct mlx5_flow_handle *rule) 85 { 86 mlx5_del_flow_rules(&rule); 87 } 88 89 struct mlx5_flow_handle * 90 mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *priv, 91 struct inpcb *inp, uint32_t tirn, uint32_t flow_tag, 92 uint16_t vlan_id) 93 { 94 struct mlx5_flow_destination dest = {}; 95 struct mlx5e_flow_table *ft = NULL; 96 #if defined(INET) || defined(INET6) 97 struct mlx5e_accel_fs_tcp *fs_tcp = &priv->fts.accel_tcp; 98 #endif 99 struct mlx5_flow_handle *flow; 100 struct mlx5_flow_spec *spec; 101 struct mlx5_flow_act flow_act = {}; 102 103 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 104 if (!spec) 105 return (ERR_PTR(-ENOMEM)); 106 107 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 108 spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG; 109 spec->flow_context.flow_tag = flow_tag; 110 111 INP_RLOCK(inp); 112 /* Set VLAN ID to match, if any. */ 113 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag); 114 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.first_vid); 115 if (vlan_id != MLX5E_ACCEL_FS_ADD_INPCB_NO_VLAN) { 116 MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.cvlan_tag); 117 MLX5_SET(fte_match_param, spec->match_value, outer_headers.first_vid, vlan_id); 118 } 119 120 /* Set TCP port numbers. */ 121 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 122 outer_headers.tcp_dport); 123 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 124 outer_headers.tcp_sport); 125 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport, 126 ntohs(inp->inp_lport)); 127 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport, 128 ntohs(inp->inp_fport)); 129 130 /* Set IP addresses. */ 131 switch (INP_SOCKAF(inp->inp_socket)) { 132 #ifdef INET 133 case AF_INET: 134 accel_fs_tcp_set_ipv4_flow(spec, inp); 135 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV4_TCP]; 136 break; 137 #endif 138 #ifdef INET6 139 case AF_INET6: 140 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0 && 141 IN6_IS_ADDR_V4MAPPED(&inp->in6p_faddr)) { 142 accel_fs_tcp_set_ipv4_flow(spec, inp); 143 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV4_TCP]; 144 } else { 145 accel_fs_tcp_set_ipv6_flow(spec, inp); 146 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV6_TCP]; 147 } 148 break; 149 #endif 150 default: 151 break; 152 } 153 INP_RUNLOCK(inp); 154 155 if (!ft) { 156 flow = ERR_PTR(-EINVAL); 157 goto out; 158 } 159 160 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 161 dest.tir_num = tirn; 162 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 163 164 flow = mlx5_add_flow_rules(ft->t, spec, &flow_act, &dest, 1); 165 out: 166 kvfree(spec); 167 return (flow); 168 } 169 170 static int 171 accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, int type) 172 { 173 static struct mlx5_flow_spec spec = {}; 174 struct mlx5_flow_destination dest = {}; 175 struct mlx5e_accel_fs_tcp *fs_tcp; 176 struct mlx5_flow_handle *rule; 177 struct mlx5_flow_act flow_act = { 178 .action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST, 179 }; 180 181 fs_tcp = &priv->fts.accel_tcp; 182 183 spec.flow_context.flags = FLOW_CONTEXT_HAS_TAG; 184 spec.flow_context.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG; 185 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 186 187 /* 188 * Traffic not matched by flow table rules should be forwarded 189 * to the next flow table in order to not be dropped by the 190 * default action. Refer to the diagram in 191 * mlx5_en_flow_table.c for more information about the order 192 * of flow tables. 193 */ 194 dest.ft = (type == MLX5E_ACCEL_FS_TCP_NUM_TYPES - 1) ? 195 ((priv->fts.ipsec_ft) ? priv->fts.ipsec_ft : priv->fts.vlan.t) : 196 fs_tcp->tables[type + 1].t; 197 198 rule = mlx5_add_flow_rules(fs_tcp->tables[type].t, &spec, &flow_act, 199 &dest, 1); 200 if (IS_ERR(rule)) 201 return (PTR_ERR(rule)); 202 203 fs_tcp->default_rules[type] = rule; 204 return (0); 205 } 206 207 #define MLX5E_ACCEL_FS_TCP_NUM_GROUPS (2) 208 #define MLX5E_ACCEL_FS_TCP_GROUP1_SIZE (BIT(16) - 1) 209 #define MLX5E_ACCEL_FS_TCP_GROUP2_SIZE (BIT(0)) 210 #define MLX5E_ACCEL_FS_TCP_TABLE_SIZE (MLX5E_ACCEL_FS_TCP_GROUP1_SIZE +\ 211 MLX5E_ACCEL_FS_TCP_GROUP2_SIZE) 212 static int 213 accel_fs_tcp_create_groups(struct mlx5e_flow_table *ft, int type) 214 { 215 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 216 void *outer_headers_c; 217 int ix = 0; 218 u32 *in; 219 int err; 220 u8 *mc; 221 222 ft->g = kcalloc(MLX5E_ACCEL_FS_TCP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL); 223 in = kvzalloc(inlen, GFP_KERNEL); 224 if (!in || !ft->g) { 225 kfree(ft->g); 226 kvfree(in); 227 return (-ENOMEM); 228 } 229 230 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 231 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers); 232 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol); 233 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version); 234 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, cvlan_tag); 235 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, first_vid); 236 237 switch (type) { 238 case MLX5E_ACCEL_FS_IPV4_TCP: 239 case MLX5E_ACCEL_FS_IPV6_TCP: 240 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport); 241 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport); 242 break; 243 default: 244 err = -EINVAL; 245 goto out; 246 } 247 248 switch (type) { 249 case MLX5E_ACCEL_FS_IPV4_TCP: 250 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 251 src_ipv4_src_ipv6.ipv4_layout.ipv4); 252 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, 253 dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 254 break; 255 case MLX5E_ACCEL_FS_IPV6_TCP: 256 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 257 src_ipv4_src_ipv6.ipv6_layout.ipv6), 258 0xff, 16); 259 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, 260 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 261 0xff, 16); 262 break; 263 default: 264 err = -EINVAL; 265 goto out; 266 } 267 268 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 269 MLX5_SET_CFG(in, start_flow_index, ix); 270 ix += MLX5E_ACCEL_FS_TCP_GROUP1_SIZE; 271 MLX5_SET_CFG(in, end_flow_index, ix - 1); 272 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 273 if (IS_ERR(ft->g[ft->num_groups])) 274 goto err; 275 ft->num_groups++; 276 277 /* Default Flow Group */ 278 memset(in, 0, inlen); 279 MLX5_SET_CFG(in, start_flow_index, ix); 280 ix += MLX5E_ACCEL_FS_TCP_GROUP2_SIZE; 281 MLX5_SET_CFG(in, end_flow_index, ix - 1); 282 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 283 if (IS_ERR(ft->g[ft->num_groups])) 284 goto err; 285 ft->num_groups++; 286 287 kvfree(in); 288 return (0); 289 290 err: 291 err = PTR_ERR(ft->g[ft->num_groups]); 292 ft->g[ft->num_groups] = NULL; 293 out: 294 kvfree(in); 295 296 return (err); 297 } 298 299 static void 300 accel_fs_tcp_destroy_groups(struct mlx5e_flow_table *ft) 301 { 302 int i; 303 304 for (i = ft->num_groups - 1; i >= 0; i--) { 305 if (!IS_ERR_OR_NULL(ft->g[i])) 306 mlx5_destroy_flow_group(ft->g[i]); 307 ft->g[i] = NULL; 308 } 309 ft->num_groups = 0; 310 } 311 312 static int 313 accel_fs_tcp_create_table(struct mlx5e_priv *priv, int type) 314 { 315 struct mlx5e_flow_table *ft = &priv->fts.accel_tcp.tables[type]; 316 struct mlx5_flow_table_attr ft_attr = {}; 317 int err; 318 319 ft->num_groups = 0; 320 ft_attr.max_fte = MLX5E_ACCEL_FS_TCP_TABLE_SIZE; 321 ft_attr.level = type; 322 ft->t = mlx5_create_flow_table(priv->fts.accel_tcp.ns, &ft_attr); 323 if (IS_ERR(ft->t)) { 324 err = PTR_ERR(ft->t); 325 ft->t = NULL; 326 return (err); 327 } 328 329 err = accel_fs_tcp_create_groups(ft, type); 330 if (err) 331 goto err_destroy_flow_table; 332 333 return (0); 334 335 err_destroy_flow_table: 336 mlx5_destroy_flow_table(ft->t); 337 ft->t = NULL; 338 return (err); 339 } 340 341 static void 342 accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i) 343 { 344 struct mlx5e_accel_fs_tcp *fs_tcp; 345 struct mlx5e_flow_table *ft; 346 347 fs_tcp = &priv->fts.accel_tcp; 348 ft = fs_tcp->tables + i; 349 350 accel_fs_tcp_destroy_groups(ft); 351 kfree(ft->g); 352 ft->g = NULL; 353 mlx5_destroy_flow_table(ft->t); 354 ft->t = NULL; 355 } 356 357 void 358 mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv) 359 { 360 int i; 361 362 if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version)) 363 return; 364 365 for (i = 0; i < MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) { 366 mlx5_del_flow_rules(&priv->fts.accel_tcp.default_rules[i]); 367 accel_fs_tcp_destroy_table(priv, i); 368 } 369 } 370 371 int 372 mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) 373 { 374 int i, err; 375 376 if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version)) 377 return (0); 378 379 /* Setup namespace pointer. */ 380 priv->fts.accel_tcp.ns = mlx5_get_flow_namespace( 381 priv->mdev, MLX5_FLOW_NAMESPACE_OFFLOADS); 382 383 /* 384 * Create flow tables first, because the priority level is 385 * assigned at allocation time. 386 */ 387 for (i = 0; i != MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) { 388 err = accel_fs_tcp_create_table(priv, i); 389 if (err) 390 goto err_destroy_tables; 391 } 392 393 /* Create default rules last. */ 394 for (i = 0; i != MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) { 395 err = accel_fs_tcp_add_default_rule(priv, i); 396 if (err) 397 goto err_destroy_rules; 398 } 399 return (0); 400 401 err_destroy_rules: 402 while (i--) 403 mlx5_del_flow_rules(&priv->fts.accel_tcp.default_rules[i]); 404 i = MLX5E_ACCEL_FS_TCP_NUM_TYPES; 405 406 err_destroy_tables: 407 while (i--) 408 accel_fs_tcp_destroy_table(priv, i); 409 return (err); 410 } 411