1 /* $NetBSD: npf_mbuf.c,v 1.11 2013/02/19 23:57:37 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2009-2012 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This material is based upon work partially supported by The 8 * NetBSD Foundation under a contract with Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * NPF network buffer management interface. 34 * 35 * Network buffer in NetBSD is mbuf. Internal mbuf structures are 36 * abstracted within this source. 37 */ 38 39 #include <sys/cdefs.h> 40 __KERNEL_RCSID(0, "$NetBSD: npf_mbuf.c,v 1.11 2013/02/19 23:57:37 rmind Exp $"); 41 42 #include <sys/param.h> 43 #include <sys/mbuf.h> 44 45 #include "npf_impl.h" 46 47 #define NBUF_ENSURE_ALIGN (MAX(COHERENCY_UNIT, 64)) 48 #define NBUF_ENSURE_MASK (NBUF_ENSURE_ALIGN - 1) 49 #define NBUF_ENSURE_ROUNDUP(x) (((x) + NBUF_ENSURE_ALIGN) & ~NBUF_ENSURE_MASK) 50 51 void 52 nbuf_init(nbuf_t *nbuf, struct mbuf *m, const ifnet_t *ifp) 53 { 54 KASSERT((m->m_flags & M_PKTHDR) != 0); 55 KASSERT(ifp != NULL); 56 57 nbuf->nb_mbuf0 = m; 58 nbuf->nb_ifp = ifp; 59 nbuf_reset(nbuf); 60 } 61 62 void 63 nbuf_reset(nbuf_t *nbuf) 64 { 65 struct mbuf *m = nbuf->nb_mbuf0; 66 67 nbuf->nb_mbuf = m; 68 nbuf->nb_nptr = mtod(m, void *); 69 } 70 71 void * 72 nbuf_dataptr(nbuf_t *nbuf) 73 { 74 KASSERT(nbuf->nb_nptr); 75 return nbuf->nb_nptr; 76 } 77 78 size_t 79 nbuf_offset(const nbuf_t *nbuf) 80 { 81 const struct mbuf *m = nbuf->nb_mbuf; 82 const u_int off = (uintptr_t)nbuf->nb_nptr - mtod(m, uintptr_t); 83 const int poff = m_length(nbuf->nb_mbuf0) - m_length(m) + off; 84 85 return poff; 86 } 87 88 struct mbuf * 89 nbuf_head_mbuf(nbuf_t *nbuf) 90 { 91 return nbuf->nb_mbuf0; 92 } 93 94 bool 95 nbuf_flag_p(const nbuf_t *nbuf, int flag) 96 { 97 return (nbuf->nb_flags & flag) != 0; 98 } 99 100 void 101 nbuf_unset_flag(nbuf_t *nbuf, int flag) 102 { 103 nbuf->nb_flags &= ~flag; 104 } 105 106 /* 107 * nbuf_advance: advance in nbuf or chain by specified amount of bytes and, 108 * if requested, ensure that the area *after* advance is contiguous. 109 * 110 * => Returns new pointer to data in nbuf or NULL if offset is invalid. 111 * => Current nbuf and the offset is stored in the nbuf metadata. 112 */ 113 void * 114 nbuf_advance(nbuf_t *nbuf, size_t len, size_t ensure) 115 { 116 struct mbuf *m = nbuf->nb_mbuf; 117 u_int off, wmark; 118 uint8_t *d; 119 120 /* Offset with amount to advance. */ 121 off = (uintptr_t)nbuf->nb_nptr - mtod(m, uintptr_t) + len; 122 wmark = m->m_len; 123 124 /* Find the mbuf according to offset. */ 125 while (__predict_false(wmark <= off)) { 126 m = m->m_next; 127 if (__predict_false(m == NULL)) { 128 /* 129 * If end of the chain, then the offset is 130 * higher than packet length. 131 */ 132 return NULL; 133 } 134 wmark += m->m_len; 135 } 136 KASSERT(off < m_length(nbuf->nb_mbuf0)); 137 138 /* Offset in mbuf data. */ 139 d = mtod(m, uint8_t *); 140 KASSERT(off >= (wmark - m->m_len)); 141 d += (off - (wmark - m->m_len)); 142 143 nbuf->nb_mbuf = m; 144 nbuf->nb_nptr = d; 145 146 if (ensure) { 147 /* Ensure contiguousness (may change nbuf chain). */ 148 d = nbuf_ensure_contig(nbuf, ensure); 149 } 150 return d; 151 } 152 153 /* 154 * nbuf_ensure_contig: check whether the specified length from the current 155 * point in the nbuf is contiguous. If not, rearrange the chain to be so. 156 * 157 * => Returns pointer to the data at the current offset in the buffer. 158 * => Returns NULL on failure and nbuf becomes invalid. 159 */ 160 void * 161 nbuf_ensure_contig(nbuf_t *nbuf, size_t len) 162 { 163 const struct mbuf * const n = nbuf->nb_mbuf; 164 const size_t off = (uintptr_t)nbuf->nb_nptr - mtod(n, uintptr_t); 165 166 KASSERT(off <= n->m_len); 167 168 if (__predict_false(n->m_len < (off + len))) { 169 struct mbuf *m = nbuf->nb_mbuf0; 170 const size_t foff = nbuf_offset(nbuf); 171 const size_t plen = m_length(m); 172 const size_t mlen = m->m_len; 173 size_t target; 174 bool success; 175 176 npf_stats_inc(NPF_STAT_NBUF_NONCONTIG); 177 178 /* Attempt to round-up to NBUF_ENSURE_ALIGN bytes. */ 179 if ((target = NBUF_ENSURE_ROUNDUP(foff + len)) > plen) { 180 target = foff + len; 181 } 182 183 /* Rearrange the chain to be contiguous. */ 184 KASSERT((m->m_flags & M_PKTHDR) != 0); 185 success = m_ensure_contig(&m, target); 186 KASSERT(m != NULL); 187 188 /* If no change in the chain: return what we have. */ 189 if (m == nbuf->nb_mbuf0 && m->m_len == mlen) { 190 return success ? nbuf->nb_nptr : NULL; 191 } 192 193 /* 194 * The mbuf chain was re-arranged. Update the pointers 195 * accordingly and indicate that the references to the data 196 * might need a reset. 197 */ 198 KASSERT((m->m_flags & M_PKTHDR) != 0); 199 nbuf->nb_mbuf0 = m; 200 nbuf->nb_mbuf = m; 201 202 KASSERT(foff < m->m_len && foff < m_length(m)); 203 nbuf->nb_nptr = mtod(m, uint8_t *) + foff; 204 nbuf->nb_flags |= NBUF_DATAREF_RESET; 205 206 if (!success) { 207 npf_stats_inc(NPF_STAT_NBUF_CONTIG_FAIL); 208 return NULL; 209 } 210 } 211 return nbuf->nb_nptr; 212 } 213 214 void * 215 nbuf_ensure_writable(nbuf_t *nbuf, size_t len) 216 { 217 struct mbuf *m = nbuf->nb_mbuf; 218 const u_int off = (uintptr_t)nbuf->nb_nptr - mtod(m, uintptr_t); 219 const int tlen = off + len; 220 bool head_buf; 221 222 KASSERT(off < m_length(nbuf->nb_mbuf0)); 223 224 if (!M_UNWRITABLE(m, tlen)) { 225 return nbuf->nb_nptr; 226 } 227 head_buf = (nbuf->nb_mbuf0 == m); 228 if (m_makewritable(&m, 0, tlen, M_NOWAIT)) { 229 memset(nbuf, 0, sizeof(nbuf_t)); 230 return NULL; 231 } 232 if (head_buf) { 233 KASSERT((m->m_flags & M_PKTHDR) != 0); 234 KASSERT(off < m_length(m)); 235 nbuf->nb_mbuf0 = m; 236 } 237 nbuf->nb_mbuf = m; 238 nbuf->nb_nptr = mtod(m, uint8_t *) + off; 239 240 return nbuf->nb_nptr; 241 } 242 243 bool 244 nbuf_cksum_barrier(nbuf_t *nbuf, int di) 245 { 246 struct mbuf *m; 247 248 if (di != PFIL_OUT) { 249 return false; 250 } 251 m = nbuf->nb_mbuf0; 252 KASSERT((m->m_flags & M_PKTHDR) != 0); 253 254 if (m->m_pkthdr.csum_flags & (M_CSUM_TCPv4 | M_CSUM_UDPv4)) { 255 in_delayed_cksum(m); 256 m->m_pkthdr.csum_flags &= ~(M_CSUM_TCPv4 | M_CSUM_UDPv4); 257 return true; 258 } 259 return false; 260 } 261 262 /* 263 * nbuf_add_tag: add a tag to specified network buffer. 264 * 265 * => Returns 0 on success or errno on failure. 266 */ 267 int 268 nbuf_add_tag(nbuf_t *nbuf, uint32_t key, uint32_t val) 269 { 270 struct mbuf *m = nbuf->nb_mbuf0; 271 struct m_tag *mt; 272 uint32_t *dat; 273 274 KASSERT((m->m_flags & M_PKTHDR) != 0); 275 276 mt = m_tag_get(PACKET_TAG_NPF, sizeof(uint32_t), M_NOWAIT); 277 if (mt == NULL) { 278 return ENOMEM; 279 } 280 dat = (uint32_t *)(mt + 1); 281 *dat = val; 282 m_tag_prepend(m, mt); 283 return 0; 284 } 285 286 /* 287 * nbuf_find_tag: find a tag in specified network buffer. 288 * 289 * => Returns 0 on success or errno on failure. 290 */ 291 int 292 nbuf_find_tag(nbuf_t *nbuf, uint32_t key, void **data) 293 { 294 struct mbuf *m = nbuf->nb_mbuf0; 295 struct m_tag *mt; 296 297 KASSERT((m->m_flags & M_PKTHDR) != 0); 298 299 mt = m_tag_find(m, PACKET_TAG_NPF, NULL); 300 if (mt == NULL) { 301 return EINVAL; 302 } 303 *data = (void *)(mt + 1); 304 return 0; 305 } 306