1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 #ifndef _RDSV3_IMPL_H
26 #define _RDSV3_IMPL_H
27
28 #include <sys/atomic.h>
29
30 /*
31 * This file is only present in Solaris
32 */
33
34 #ifdef __cplusplus
35 extern "C" {
36 #endif
37
38 extern dev_info_t *rdsv3_dev_info;
39
40 #define uint16_be_t uint16_t
41 #define uint32_be_t uint32_t
42 #define uint64_be_t uint64_t
43
44 /*
45 * RDS Well known service id
46 * Format: 0x1h00144Fhhhhhhhh
47 * "00144F" is the Sun OUI
48 * 'h' can be any hex-decimal digit.
49 */
50 #define RDS_SERVICE_ID 0x1000144F00000001ULL
51
52 /*
53 * Atomic operations
54 */
55 typedef unsigned int atomic_t;
56 #define ATOMIC_INIT(a) a
57
58 #define atomic_get(p) (*(p))
59
60 #define atomic_cmpset_long(p, c, n) \
61 ((c == atomic_cas_uint(p, c, n)) ? c : -1)
62
63 #define atomic_dec_and_test(a) \
64 (atomic_dec_uint_nv((a)) == 0)
65
66 #define atomic_cmpxchg(a, o, n) \
67 atomic_cas_uint(a, o, n)
68
69 #ifdef _LP64
70 #define set_bit(b, p) \
71 atomic_or_ulong(((volatile ulong_t *)(void *)(p)) + ((b) >> 6), \
72 1ul << ((b) & 0x3f))
73
74 #define clear_bit(b, p) \
75 atomic_and_ulong(((volatile ulong_t *)(void *)(p)) + ((b) >> 6), \
76 ~(1ul << ((b) & 0x3f)))
77
78 #define test_bit(b, p) \
79 (((volatile ulong_t *)(void *)(p))[(b) >> 6] & (1ul << ((b) & 0x3f)))
80
81 #define test_and_set_bit(b, p) \
82 atomic_set_long_excl(((ulong_t *)(void *)(p)) + \
83 ((b) >> 6), ((b) & 0x3f))
84 #define test_and_clear_bit(b, p) \
85 !atomic_clear_long_excl(((ulong_t *)(void *)(p)) + ((b) >> 6), \
86 ((b) & 0x3f))
87 #else
88 #define set_bit(b, p) \
89 atomic_or_uint(((volatile uint_t *)(void *)p) + (b >> 5), \
90 1ul << (b & 0x1f))
91
92 #define clear_bit(b, p) \
93 atomic_and_uint(((volatile uint_t *)(void *)p) + (b >> 5), \
94 ~(1ul << (b & 0x1f)))
95
96 #define test_bit(b, p) \
97 (((volatile uint_t *)(void *)p)[b >> 5] & (1ul << (b & 0x1f)))
98
99 #define test_and_set_bit(b, p) \
100 atomic_set_long_excl(((ulong_t *)(void *)p) + (b >> 5), (b & 0x1f))
101 #define test_and_clear_bit(b, p) \
102 !atomic_clear_long_excl(((ulong_t *)(void *)p) + (b >> 5), (b & 0x1f))
103 #endif
104
105 /*
106 * These macros and/or constants are used instead of Linux
107 * generic_{test,__{clear,set}}_le_bit().
108 */
109 #if defined(sparc)
110 #define LE_BIT_XOR ((BITS_PER_LONG-1) & ~0x7)
111 #else
112 #define LE_BIT_XOR 0
113 #endif
114
115 #define set_le_bit(b, p) set_bit(b ^ LE_BIT_XOR, p)
116 #define clear_le_bit(b, p) clear_bit(b ^ LE_BIT_XOR, p)
117 #define test_le_bit(b, p) test_bit(b ^ LE_BIT_XOR, p)
118
119 uint_t rdsv3_one_sec_in_hz;
120
121 #define jiffies 100
122 #define HZ (drv_hztousec(1))
123 #define container_of(m, s, name) \
124 (void *)((uintptr_t)(m) - (uintptr_t)offsetof(s, name))
125 #define ARRAY_SIZE(x) (sizeof (x) / sizeof (x[0]))
126 /* setting this to PAGESIZE throws build errors */
127 #define PAGE_SIZE 4096 /* xxx - fix this */
128 #define BITS_PER_LONG (sizeof (unsigned long) * 8)
129
130 /* debug */
131 #define RDSV3_PANIC() cmn_err(CE_PANIC, "Panic forced by RDSV3");
132
133 /* ERR */
134 #define MAX_ERRNO 4095
135 #define ERR_PTR(x) ((void *)(uintptr_t)x)
136 #define IS_ERR(ptr) (((uintptr_t)ptr) >= (uintptr_t)-MAX_ERRNO)
137 #define PTR_ERR(ptr) (int)(uintptr_t)ptr
138
139 #define MAX_SCHEDULE_TIMEOUT (~0UL>>1)
140
141 #define RDMA_CM_EVENT_ADDR_CHANGE 14
142
143 /* list */
144 /* copied and modified list_remove_node */
145 #define list_remove_node(node) \
146 if ((node)->list_next != NULL) { \
147 (node)->list_prev->list_next = (node)->list_next; \
148 (node)->list_next->list_prev = (node)->list_prev; \
149 (node)->list_next = (node)->list_prev = NULL; \
150 }
151
152 #define list_splice(src, dst) { \
153 list_create(dst, (src)->list_size, (src)->list_offset); \
154 list_move_tail(dst, src); \
155 }
156
157 #define RDSV3_FOR_EACH_LIST_NODE(objp, listp, member) \
158 for (objp = list_head(listp); objp; objp = list_next(listp, objp))
159 #define RDSV3_FOR_EACH_LIST_NODE_SAFE(objp, tmp, listp, member) \
160 for (objp = list_head(listp), tmp = (objp != NULL) ? \
161 list_next(listp, objp) : NULL; \
162 objp; \
163 objp = tmp, tmp = (objp != NULL) ? \
164 list_next(listp, objp) : NULL)
165
166 /* simulate wait_queue_head_t */
167 typedef struct rdsv3_wait_queue_s {
168 kmutex_t waitq_mutex;
169 kcondvar_t waitq_cv;
170 uint_t waitq_waiters;
171 } rdsv3_wait_queue_t;
172
173 #define rdsv3_init_waitqueue(waitqp) \
174 mutex_init(&(waitqp)->waitq_mutex, NULL, MUTEX_DRIVER, NULL); \
175 cv_init(&(waitqp)->waitq_cv, NULL, CV_DRIVER, NULL); \
176 (waitqp)->waitq_waiters = 0
177
178 #define rdsv3_exit_waitqueue(waitqp) \
179 ASSERT((waitqp)->waitq_waiters == 0); \
180 mutex_destroy(&(waitqp)->waitq_mutex); \
181 cv_destroy(&(waitqp)->waitq_cv)
182
183 #define rdsv3_wake_up(waitqp) { \
184 mutex_enter(&(waitqp)->waitq_mutex); \
185 if ((waitqp)->waitq_waiters) \
186 cv_signal(&(waitqp)->waitq_cv); \
187 mutex_exit(&(waitqp)->waitq_mutex); \
188 }
189
190 #define rdsv3_wake_up_all(waitqp) { \
191 mutex_enter(&(waitqp)->waitq_mutex); \
192 if ((waitqp)->waitq_waiters) \
193 cv_broadcast(&(waitqp)->waitq_cv); \
194 mutex_exit(&(waitqp)->waitq_mutex); \
195 }
196
197 /* analogous to cv_wait */
198 #define rdsv3_wait_event(waitq, condition) \
199 { \
200 mutex_enter(&(waitq)->waitq_mutex); \
201 (waitq)->waitq_waiters++; \
202 while (!(condition)) { \
203 cv_wait(&(waitq)->waitq_cv, &(waitq)->waitq_mutex); \
204 } \
205 (waitq)->waitq_waiters--; \
206 mutex_exit(&(waitq)->waitq_mutex); \
207 }
208
209 /* analogous to cv_wait_sig */
210 #define rdsv3_wait_sig(waitqp, condition) \
211 ( \
212 { \
213 int cv_return = 1; \
214 mutex_enter(&(waitqp)->waitq_mutex); \
215 (waitqp)->waitq_waiters++; \
216 while (!(condition)) { \
217 cv_return = cv_wait_sig(&(waitqp)->waitq_cv, \
218 &(waitqp)->waitq_mutex); \
219 if (cv_return == 0) { \
220 break; \
221 } \
222 } \
223 (waitqp)->waitq_waiters--; \
224 mutex_exit(&(waitqp)->waitq_mutex); \
225 cv_return; \
226 } \
227 )
228
229 #define SOCK_DEAD 1ul
230
231 /* socket */
232 typedef struct rsock {
233 sock_upper_handle_t sk_upper_handle;
234 sock_upcalls_t *sk_upcalls;
235
236 kmutex_t sk_lock;
237 ulong_t sk_flag;
238 rdsv3_wait_queue_t *sk_sleep; /* Also protected by rs_recv_lock */
239 int sk_sndbuf;
240 int sk_rcvbuf;
241 atomic_t sk_refcount;
242
243 struct rdsv3_sock *sk_protinfo;
244 } rsock_t;
245
246 typedef struct rdsv3_conn_info_s {
247 uint32_be_t c_laddr;
248 uint32_be_t c_faddr;
249 } rdsv3_conn_info_t;
250
251 /* WQ */
252 typedef struct rdsv3_workqueue_struct_s {
253 kmutex_t wq_lock;
254 uint_t wq_state;
255 int wq_pending;
256 list_t wq_queue;
257 } rdsv3_workqueue_struct_t;
258
259 struct rdsv3_work_s;
260 typedef void (*rdsv3_work_func_t)(struct rdsv3_work_s *);
261 typedef struct rdsv3_work_s {
262 list_node_t work_item;
263 rdsv3_work_func_t func;
264 } rdsv3_work_t;
265
266 /* simulate delayed_work */
267 typedef struct rdsv3_delayed_work_s {
268 kmutex_t lock;
269 rdsv3_work_t work;
270 timeout_id_t timeid;
271 rdsv3_workqueue_struct_t *wq;
272 } rdsv3_delayed_work_t;
273
274 #define RDSV3_INIT_WORK(wp, f) (wp)->func = f
275 #define RDSV3_INIT_DELAYED_WORK(dwp, f) \
276 (dwp)->work.func = f; \
277 mutex_init(&(dwp)->lock, NULL, MUTEX_DRIVER, NULL); \
278 (dwp)->timeid = 0
279
280 /* simulate scatterlist */
281 struct rdsv3_scatterlist {
282 caddr_t vaddr;
283 uint_t length;
284 ibt_wr_ds_t *sgl;
285 ibt_mi_hdl_t mihdl;
286 };
287 #define rdsv3_sg_page(scat) (scat)->vaddr
288 #define rdsv3_sg_len(scat) (scat)->length
289 #define rdsv3_sg_set_page(scat, pg, len, off) \
290 (scat)->vaddr = (caddr_t)(pg + off); \
291 (scat)->length = len
292 #define rdsv3_ib_sg_dma_len(dev, scat) rdsv3_sg_len(scat)
293
294 /* copied from sys/socket.h */
295 #if defined(__sparc)
296 /* To maintain backward compatibility, alignment needs to be 8 on sparc. */
297 #define _CMSG_HDR_ALIGNMENT 8
298 #else
299 /* for __i386 (and other future architectures) */
300 #define _CMSG_HDR_ALIGNMENT 4
301 #endif /* defined(__sparc) */
302
303 /*
304 * The cmsg headers (and macros dealing with them) were made available as
305 * part of UNIX95 and hence need to be protected with a _XPG4_2 define.
306 */
307 #define _CMSG_DATA_ALIGNMENT (sizeof (int))
308 #define _CMSG_HDR_ALIGN(x) (((uintptr_t)(x) + _CMSG_HDR_ALIGNMENT - 1) & \
309 ~(_CMSG_HDR_ALIGNMENT - 1))
310 #define _CMSG_DATA_ALIGN(x) (((uintptr_t)(x) + _CMSG_DATA_ALIGNMENT - 1) & \
311 ~(_CMSG_DATA_ALIGNMENT - 1))
312 #define CMSG_DATA(c) \
313 ((unsigned char *)_CMSG_DATA_ALIGN((struct cmsghdr *)(c) + 1))
314
315 #define CMSG_FIRSTHDR(m) \
316 (((m)->msg_controllen < sizeof (struct cmsghdr)) ? \
317 (struct cmsghdr *)0 : (struct cmsghdr *)((m)->msg_control))
318
319 #define CMSG_NXTHDR(m, c) \
320 (((c) == 0) ? CMSG_FIRSTHDR(m) : \
321 ((((uintptr_t)_CMSG_HDR_ALIGN((char *)(c) + \
322 ((struct cmsghdr *)(c))->cmsg_len) + sizeof (struct cmsghdr)) > \
323 (((uintptr_t)((struct msghdr *)(m))->msg_control) + \
324 ((uintptr_t)((struct msghdr *)(m))->msg_controllen))) ? \
325 ((struct cmsghdr *)0) : \
326 ((struct cmsghdr *)_CMSG_HDR_ALIGN((char *)(c) + \
327 ((struct cmsghdr *)(c))->cmsg_len))))
328
329 /* Amount of space + padding needed for a message of length l */
330 #define CMSG_SPACE(l) \
331 ((unsigned int)_CMSG_HDR_ALIGN(sizeof (struct cmsghdr) + (l)))
332
333 /* Value to be used in cmsg_len, does not include trailing padding */
334 #define CMSG_LEN(l) \
335 ((unsigned int)_CMSG_DATA_ALIGN(sizeof (struct cmsghdr)) + (l))
336
337 /* OFUV -> IB */
338 #define RDSV3_IBDEV2HCAHDL(device) (device)->hca_hdl
339 #define RDSV3_QP2CHANHDL(qp) (qp)->ibt_qp
340 #define RDSV3_PD2PDHDL(pd) (pd)->ibt_pd
341 #define RDSV3_CQ2CQHDL(cq) (cq)->ibt_cq
342
343 struct rdsv3_hdrs_mr {
344 ibt_lkey_t lkey;
345 caddr_t addr;
346 size_t size;
347 ibt_mr_hdl_t hdl;
348 };
349
350 /* rdsv3_impl.c */
351 void rdsv3_trans_init();
352 boolean_t rdsv3_capable_interface(struct lifreq *lifrp);
353 int rdsv3_do_ip_ioctl(ksocket_t so4, void **ipaddrs, int *size, int *nifs);
354 int rdsv3_do_ip_ioctl_old(ksocket_t so4, void **ipaddrs, int *size, int *nifs);
355 boolean_t rdsv3_isloopback(ipaddr_t addr);
356 void rdsv3_cancel_delayed_work(rdsv3_delayed_work_t *dwp);
357 void rdsv3_flush_workqueue(rdsv3_workqueue_struct_t *wq);
358 void rdsv3_queue_work(rdsv3_workqueue_struct_t *wq, rdsv3_work_t *wp);
359 void rdsv3_queue_delayed_work(rdsv3_workqueue_struct_t *wq,
360 rdsv3_delayed_work_t *dwp, uint_t delay);
361 struct rsock *rdsv3_sk_alloc();
362 void rdsv3_sock_init_data(struct rsock *sk);
363 void rdsv3_sock_exit_data(struct rsock *sk);
364 void rdsv3_destroy_task_workqueue(rdsv3_workqueue_struct_t *wq);
365 rdsv3_workqueue_struct_t *rdsv3_create_task_workqueue(char *name);
366 int rdsv3_conn_constructor(void *buf, void *arg, int kmflags);
367 void rdsv3_conn_destructor(void *buf, void *arg);
368 int rdsv3_conn_compare(const void *conn1, const void *conn2);
369 void rdsv3_loop_init();
370 int rdsv3_mr_compare(const void *mr1, const void *mr2);
371 int rdsv3_put_cmsg(struct nmsghdr *msg, int level, int type, size_t size,
372 void *payload);
373 int rdsv3_verify_bind_address(ipaddr_t addr);
374 uint16_t rdsv3_ip_fast_csum(void *buffer, size_t length);
375 uint_t rdsv3_ib_dma_map_sg(struct ib_device *dev, struct rdsv3_scatterlist
376 *scat, uint_t num);
377 void rdsv3_ib_dma_unmap_sg(ib_device_t *dev, struct rdsv3_scatterlist *scat,
378 uint_t num);
379 static inline void
rdsv3_sk_sock_hold(struct rsock * sk)380 rdsv3_sk_sock_hold(struct rsock *sk)
381 {
382 atomic_add_32(&sk->sk_refcount, 1);
383 }
384 static inline void
rdsv3_sk_sock_put(struct rsock * sk)385 rdsv3_sk_sock_put(struct rsock *sk)
386 {
387 if (atomic_dec_and_test(&sk->sk_refcount))
388 rdsv3_sock_exit_data(sk);
389 }
390 static inline int
rdsv3_sk_sock_flag(struct rsock * sk,uint_t flag)391 rdsv3_sk_sock_flag(struct rsock *sk, uint_t flag)
392 {
393 return (test_bit(flag, &sk->sk_flag));
394 }
395 static inline void
rdsv3_sk_sock_orphan(struct rsock * sk)396 rdsv3_sk_sock_orphan(struct rsock *sk)
397 {
398 set_bit(SOCK_DEAD, &sk->sk_flag);
399 }
400
401 #define rdsv3_sndtimeo(a, b) b ? 0 : 3600 /* check this value on linux */
402 #define rdsv3_rcvtimeo(a, b) b ? 0 : 3600 /* check this value on linux */
403
404 void rdsv3_ib_free_conn(void *arg);
405
406 #ifdef __cplusplus
407 }
408 #endif
409
410 #endif /* _RDSV3_IMPL_H */
411