1 /*
2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 */
35
36 /* AUTHOR Edward Bortnikov
37 *
38 * DESCRIPTION
39 * The lower-level MAD transport interface implementation
40 * that allows sending a single MAD/receiving a callback
41 * when a single MAD is received.
42 */
43
44 #if HAVE_CONFIG_H
45 # include <config.h>
46 #endif /* HAVE_CONFIG_H */
47
48 #include <sys/types.h>
49 #include <sys/stat.h>
50 #include <sys/ioctl.h>
51 #include <fcntl.h>
52 #include <stddef.h>
53 #include <errno.h>
54 #include <stdlib.h>
55 #include <string.h>
56
57 #include <vendor/osm_vendor_api.h>
58 #include <vendor/osm_vendor_mlx_transport.h>
59 #include <vendor/osm_vendor_mlx_transport_anafa.h>
60 #include <vendor/osm_vendor_mlx_dispatcher.h>
61 #include <vendor/osm_vendor_mlx_svc.h>
62 #include <vendor/osm_ts_useraccess.h>
63
64 static void
65 __osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
66 IN struct ib_mad *p_mad,
67 IN uint8_t is_smi,
68 OUT osm_mad_addr_t * p_mad_addr);
69
70 static void
71 __osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(IN const osm_mad_addr_t *
72 p_mad_addr, IN uint8_t is_smi,
73 OUT struct ib_mad *p_mad);
74
__osmv_TOPSPIN_ANAFA_receiver_thr(void * p_ctx)75 void __osmv_TOPSPIN_ANAFA_receiver_thr(void *p_ctx)
76 {
77 int ts_ret_code;
78 struct ib_mad mad;
79 osm_mad_addr_t mad_addr;
80 osmv_bind_obj_t *const p_bo = (osmv_bind_obj_t *) p_ctx;
81 ib_api_status_t status = IB_SUCCESS;
82
83 OSM_LOG_ENTER(p_bo->p_vendor->p_log);
84
85 /* Make sure the p_bo object is still relevant */
86 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
87 return;
88
89 /* we set the type of cancelation for this thread */
90 /* pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); */
91
92 while (1) {
93 /* Make sure the p_bo object is still relevant */
94 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
95 return;
96
97 /* we read one mad at a time and pass it to the read callback function */
98 ts_ret_code =
99 read(((osmv_TOPSPIN_ANAFA_transport_mgr_t *) (p_bo->
100 p_transp_mgr))->
101 device_fd, &mad, sizeof(mad));
102
103 /* Make sure the p_bo object is still relevant */
104 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
105 return;
106
107 if (ts_ret_code != sizeof(mad)) {
108 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
109 "__osmv_TOPSPIN_ANAFA_receiver_thr: ERR 6903: "
110 "error with read, bytes = %d\n", ts_ret_code);
111 break;
112 } else {
113 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
114 "__osmv_TOPSPIN_ANAFA_receiver_thr: "
115 "MAD QPN:%d SLID:0x%04x class:0x%02x "
116 "method:0x%02x attr:0x%04x status:0x%04x "
117 "tid:0x%016" PRIx64 "\n",
118 mad.dqpn,
119 cl_ntoh16(mad.slid),
120 mad.mgmt_class,
121 mad.r_method,
122 cl_ntoh16(mad.attribute_id),
123 cl_ntoh16(mad.status),
124 cl_ntoh64(mad.transaction_id));
125
126 /* first arrange an address */
127 __osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr
128 (p_bo->p_vendor, &mad,
129 (((ib_mad_t *) & mad)->mgmt_class ==
130 IB_MCLASS_SUBN_LID)
131 || (((ib_mad_t *) & mad)->mgmt_class ==
132 IB_MCLASS_SUBN_DIR), &mad_addr);
133
134 /* call the receiver callback */
135
136 status =
137 osmv_dispatch_mad((osm_bind_handle_t) p_bo,
138 (void *)&mad, &mad_addr);
139
140 /* Make sure the p_bo object is still relevant */
141 if (p_bo->magic_ptr != p_bo)
142 return;
143
144 if (IB_INTERRUPTED == status) {
145
146 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
147 "__osmv_TOPSPIN_ANAFA_receiver_thr: "
148 "The bind handle %p is being closed. "
149 "Breaking the loop.\n", p_bo);
150 break;
151 }
152 }
153 }
154
155 OSM_LOG_EXIT(p_bo->p_vendor->p_log);
156 }
157
158 /*
159 * NAME
160 * osmv_transport_init
161 *
162 * DESCRIPTION
163 * Setup the MAD transport infrastructure (filters, callbacks etc).
164 */
165
166 ib_api_status_t
osmv_transport_init(IN osm_bind_info_t * p_info,IN char hca_id[VENDOR_HCA_MAXNAMES],IN uint8_t hca_idx,IN osmv_bind_obj_t * p_bo)167 osmv_transport_init(IN osm_bind_info_t * p_info,
168 IN char hca_id[VENDOR_HCA_MAXNAMES],
169 IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo)
170 {
171 cl_status_t cl_st;
172
173 int ts_ioctl_ret;
174 int device_fd;
175 char *device_file = "/dev/ts_ua0";
176 osm_ts_user_mad_filter filter;
177 osmv_TOPSPIN_ANAFA_transport_mgr_t *p_mgr;
178 osmv_TOPSPIN_ANAFA_transport_info_t *p_tpot_info;
179 p_tpot_info =
180 (osmv_TOPSPIN_ANAFA_transport_info_t *) p_bo->p_vendor->
181 p_transport_info;
182
183 p_mgr = malloc(sizeof(osmv_TOPSPIN_ANAFA_transport_mgr_t));
184 if (!p_mgr) {
185 return IB_INSUFFICIENT_MEMORY;
186 }
187
188 memset(p_mgr, 0, sizeof(osmv_TOPSPIN_ANAFA_transport_mgr_t));
189
190 /* open TopSpin file device */
191 device_fd = open(device_file, O_RDWR);
192 if (device_fd < 0) {
193 fprintf(stderr, "Fatal: Fail to open the file:%s err:%d\n",
194 device_file, errno);
195 return IB_ERROR;
196 }
197 p_mgr->device_fd = device_fd;
198
199 /*
200 * Create the MAD filter on this file handle.
201 */
202
203 filter.port = 0; /* Victor */
204 filter.direction = TS_IB_MAD_DIRECTION_IN;
205 filter.mask =
206 TS_IB_MAD_FILTER_DIRECTION |
207 TS_IB_MAD_FILTER_PORT |
208 TS_IB_MAD_FILTER_QPN | TS_IB_MAD_FILTER_MGMT_CLASS;
209
210 switch (p_info->mad_class) {
211 case IB_MCLASS_SUBN_LID:
212 case IB_MCLASS_SUBN_DIR:
213 filter.qpn = 0;
214 filter.mgmt_class = IB_MCLASS_SUBN_LID;
215 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter);
216 if (ts_ioctl_ret < 0) {
217 return IB_ERROR;
218 }
219
220 filter.mgmt_class = IB_MCLASS_SUBN_DIR;
221 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter);
222 if (ts_ioctl_ret < 0) {
223 return IB_ERROR;
224 }
225
226 break;
227
228 case IB_MCLASS_SUBN_ADM:
229 default:
230 filter.qpn = 1;
231 filter.mgmt_class = p_info->mad_class;
232 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter);
233 if (ts_ioctl_ret < 0) {
234 return IB_ERROR;
235 }
236 break;
237 }
238
239 p_bo->p_transp_mgr = p_mgr;
240
241 /* Initialize the magic_ptr to the pointer of the p_bo info.
242 This will be used to signal when the object is being destroyed, so no
243 real action will be done then. */
244 p_bo->magic_ptr = p_bo;
245
246 /* init receiver thread */
247 cl_st =
248 cl_thread_init(&p_mgr->receiver, __osmv_TOPSPIN_ANAFA_receiver_thr,
249 (void *)p_bo, "osmv TOPSPIN_ANAFA rcv thr");
250
251 return (ib_api_status_t) cl_st;
252 }
253
254 /*
255 * NAME
256 * osmv_transport_send_mad
257 *
258 * DESCRIPTION
259 * Send a single MAD (256 byte)
260 */
261
262 ib_api_status_t
osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,IN void * p_mad,IN const osm_mad_addr_t * p_mad_addr)263 osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,
264 IN void *p_mad, IN const osm_mad_addr_t * p_mad_addr)
265 {
266
267 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
268 osm_vendor_t const *p_vend = p_bo->p_vendor;
269 struct ib_mad ts_mad = { 0 };
270 int ret;
271 ib_api_status_t status;
272
273 const ib_mad_t *p_mad_hdr = p_mad;
274
275 OSM_LOG_ENTER(p_vend->p_log);
276
277 /* Make sure the p_bo object is still relevant */
278 if (p_bo->magic_ptr != p_bo)
279 return IB_INVALID_CALLBACK;
280
281 /*
282 * Copy the MAD over to the sent mad
283 */
284 memcpy(&ts_mad, p_mad_hdr, MAD_BLOCK_SIZE);
285
286 /*
287 * For all sends other than directed route SM MADs,
288 * acquire an address vector for the destination.
289 */
290 if (p_mad_hdr->mgmt_class != IB_MCLASS_SUBN_DIR) {
291
292 __osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(p_mad_addr,
293 p_mad_hdr->
294 mgmt_class ==
295 IB_MCLASS_SUBN_LID,
296 &ts_mad);
297 } else {
298 /* is a directed route - we need to construct a permissive address */
299 /* we do not need port number since it is part of the mad_hndl */
300 ts_mad.dlid = IB_LID_PERMISSIVE;
301 ts_mad.slid = IB_LID_PERMISSIVE;
302 }
303 if ((p_mad_hdr->mgmt_class == IB_MCLASS_SUBN_DIR) ||
304 (p_mad_hdr->mgmt_class == IB_MCLASS_SUBN_LID)) {
305 ts_mad.sqpn = 0;
306 ts_mad.dqpn = 0;
307 } else {
308 ts_mad.sqpn = 1;
309 ts_mad.dqpn = 1;
310 }
311
312 /* ts_mad.port = p_bo->port_num; */
313 ts_mad.port = 0; /* Victor */
314
315 /* send it */
316 ret =
317 write(((osmv_TOPSPIN_ANAFA_transport_mgr_t *) (p_bo->
318 p_transp_mgr))->
319 device_fd, &ts_mad, sizeof(ts_mad));
320
321 if (ret != sizeof(ts_mad)) {
322 osm_log(p_vend->p_log, OSM_LOG_ERROR,
323 "osmv_transport_mad_send: ERR 6904: "
324 "Error sending mad (%d).\n", ret);
325 status = IB_ERROR;
326 goto Exit;
327 }
328
329 status = IB_SUCCESS;
330
331 Exit:
332 OSM_LOG_EXIT(p_vend->p_log);
333 return (status);
334 }
335
osmv_transport_done(IN const osm_bind_handle_t h_bind)336 void osmv_transport_done(IN const osm_bind_handle_t h_bind)
337 {
338 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
339 osmv_TOPSPIN_ANAFA_transport_mgr_t *p_tpot_mgr =
340 (osmv_TOPSPIN_ANAFA_transport_mgr_t *) (p_bo->p_transp_mgr);
341
342 CL_ASSERT(p_bo);
343
344 /* First of all - zero out the magic_ptr, so if a callback is called -
345 it'll know that we are currently closing down, and will not handle the
346 mad. */
347 p_bo->magic_ptr = 0;
348
349 /* usleep(3000000); */
350
351 /* pthread_cancel (p_tpot_mgr->receiver.osd.id); */
352 cl_thread_destroy(&(p_tpot_mgr->receiver));
353 free(p_tpot_mgr);
354 }
355
356 static void
__osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr,IN uint8_t is_smi,OUT struct ib_mad * p_mad)357 __osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr,
358 IN uint8_t is_smi,
359 OUT struct ib_mad *p_mad)
360 {
361
362 /* For global destination or Multicast address: */
363 p_mad->dlid = cl_ntoh16(p_mad_addr->dest_lid);
364 p_mad->sl = p_mad_addr->addr_type.gsi.service_level;
365 if (is_smi) {
366 p_mad->sqpn = 0;
367 p_mad->dqpn = 0;
368 } else {
369 p_mad->sqpn = 1;
370 p_mad->dqpn = p_mad_addr->addr_type.gsi.remote_qp;
371 }
372 /*
373 HACK we limit to the first PKey Index assuming it will
374 always be the default PKey
375 */
376 p_mad->pkey_index = 0;
377 }
378
379 static void
__osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr(IN osm_vendor_t const * p_vend,IN struct ib_mad * p_mad,IN uint8_t is_smi,OUT osm_mad_addr_t * p_mad_addr)380 __osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
381 IN struct ib_mad *p_mad,
382 IN uint8_t is_smi,
383 OUT osm_mad_addr_t * p_mad_addr)
384 {
385 p_mad_addr->dest_lid = cl_hton16(p_mad->slid);
386 p_mad_addr->static_rate = 0;
387 p_mad_addr->path_bits = 0;
388 if (is_smi) {
389 /* SMI */
390 p_mad_addr->addr_type.smi.source_lid = cl_hton16(p_mad->slid);
391 p_mad_addr->addr_type.smi.port_num = p_mad->port;
392 } else {
393 /* GSI */
394 p_mad_addr->addr_type.gsi.remote_qp = p_mad->sqpn;
395 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
396 p_mad_addr->addr_type.gsi.pkey_ix = p_mad->pkey_index;
397 p_mad_addr->addr_type.gsi.service_level = p_mad->sl;
398
399 p_mad_addr->addr_type.gsi.global_route = FALSE;
400 /* copy the GRH data if relevant - TopSpin imp doesnt relate to GRH!!! */
401 /*
402 if (p_mad_addr->addr_type.gsi.global_route)
403 {
404 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
405 ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version,
406 p_rcv_desc->grh.traffic_class,
407 p_rcv_desc->grh.flow_label);
408 p_mad_addr->addr_type.gsi.grh_info.hop_limit = p_rcv_desc->grh.hop_limit;
409 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
410 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
411 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
412 p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
413 }
414 */
415 }
416 }
417