1 /* $NetBSD: tls.c,v 1.15 2023/02/10 08:12:48 skrll Exp $ */ 2 /*- 3 * Copyright (c) 2011 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Joerg Sonnenberger. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __RCSID("$NetBSD: tls.c,v 1.15 2023/02/10 08:12:48 skrll Exp $"); 33 34 #include <sys/param.h> 35 #include <sys/ucontext.h> 36 #include <lwp.h> 37 #include <stdalign.h> 38 #include <stddef.h> 39 #include <string.h> 40 #include "debug.h" 41 #include "rtld.h" 42 43 #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) 44 45 static struct tls_tcb *_rtld_tls_allocate_locked(void); 46 47 #ifndef TLS_DTV_OFFSET 48 #define TLS_DTV_OFFSET 0 49 #endif 50 51 static size_t _rtld_tls_static_space; /* Static TLS space allocated */ 52 static size_t _rtld_tls_static_offset; /* Next offset for static TLS to use */ 53 size_t _rtld_tls_dtv_generation = 1; 54 size_t _rtld_tls_max_index = 1; 55 56 #define DTV_GENERATION(dtv) ((size_t)((dtv)[0])) 57 #define DTV_MAX_INDEX(dtv) ((size_t)((dtv)[-1])) 58 #define SET_DTV_GENERATION(dtv, val) (dtv)[0] = (void *)(size_t)(val) 59 #define SET_DTV_MAX_INDEX(dtv, val) (dtv)[-1] = (void *)(size_t)(val) 60 61 void * 62 _rtld_tls_get_addr(void *tls, size_t idx, size_t offset) 63 { 64 struct tls_tcb *tcb = tls; 65 void **dtv, **new_dtv; 66 sigset_t mask; 67 68 _rtld_exclusive_enter(&mask); 69 70 dtv = tcb->tcb_dtv; 71 72 if (__predict_false(DTV_GENERATION(dtv) != _rtld_tls_dtv_generation)) { 73 size_t to_copy = DTV_MAX_INDEX(dtv); 74 75 new_dtv = xcalloc((2 + _rtld_tls_max_index) * sizeof(*dtv)); 76 ++new_dtv; 77 if (to_copy > _rtld_tls_max_index) 78 to_copy = _rtld_tls_max_index; 79 memcpy(new_dtv + 1, dtv + 1, to_copy * sizeof(*dtv)); 80 xfree(dtv - 1); 81 dtv = tcb->tcb_dtv = new_dtv; 82 SET_DTV_MAX_INDEX(dtv, _rtld_tls_max_index); 83 SET_DTV_GENERATION(dtv, _rtld_tls_dtv_generation); 84 } 85 86 if (__predict_false(dtv[idx] == NULL)) 87 dtv[idx] = _rtld_tls_module_allocate(idx); 88 89 _rtld_exclusive_exit(&mask); 90 91 return (uint8_t *)dtv[idx] + offset; 92 } 93 94 void 95 _rtld_tls_initial_allocation(void) 96 { 97 struct tls_tcb *tcb; 98 99 _rtld_tls_static_space = _rtld_tls_static_offset + 100 RTLD_STATIC_TLS_RESERVATION; 101 102 #ifndef __HAVE_TLS_VARIANT_I 103 _rtld_tls_static_space = roundup2(_rtld_tls_static_space, 104 alignof(max_align_t)); 105 #endif 106 dbg(("_rtld_tls_static_space %zu", _rtld_tls_static_space)); 107 108 tcb = _rtld_tls_allocate_locked(); 109 #ifdef __HAVE___LWP_SETTCB 110 __lwp_settcb(tcb); 111 #else 112 _lwp_setprivate(tcb); 113 #endif 114 } 115 116 static struct tls_tcb * 117 _rtld_tls_allocate_locked(void) 118 { 119 Obj_Entry *obj; 120 struct tls_tcb *tcb; 121 uint8_t *p, *q; 122 123 p = xcalloc(_rtld_tls_static_space + sizeof(struct tls_tcb)); 124 #ifdef __HAVE_TLS_VARIANT_I 125 tcb = (struct tls_tcb *)p; 126 p += sizeof(struct tls_tcb); 127 #else 128 p += _rtld_tls_static_space; 129 tcb = (struct tls_tcb *)p; 130 tcb->tcb_self = tcb; 131 #endif 132 dbg(("tcb %p", tcb)); 133 tcb->tcb_dtv = xcalloc(sizeof(*tcb->tcb_dtv) * (2 + _rtld_tls_max_index)); 134 ++tcb->tcb_dtv; 135 SET_DTV_MAX_INDEX(tcb->tcb_dtv, _rtld_tls_max_index); 136 SET_DTV_GENERATION(tcb->tcb_dtv, _rtld_tls_dtv_generation); 137 138 for (obj = _rtld_objlist; obj != NULL; obj = obj->next) { 139 if (obj->tls_done) { 140 #ifdef __HAVE_TLS_VARIANT_I 141 q = p + obj->tlsoffset; 142 #else 143 q = p - obj->tlsoffset; 144 #endif 145 dbg(("obj %p dtv %p tlsoffset %zu", 146 obj, q, obj->tlsoffset)); 147 if (obj->tlsinitsize) 148 memcpy(q, obj->tlsinit, obj->tlsinitsize); 149 tcb->tcb_dtv[obj->tlsindex] = q; 150 } 151 } 152 153 return tcb; 154 } 155 156 struct tls_tcb * 157 _rtld_tls_allocate(void) 158 { 159 struct tls_tcb *tcb; 160 sigset_t mask; 161 162 _rtld_exclusive_enter(&mask); 163 tcb = _rtld_tls_allocate_locked(); 164 _rtld_exclusive_exit(&mask); 165 166 return tcb; 167 } 168 169 void 170 _rtld_tls_free(struct tls_tcb *tcb) 171 { 172 size_t i, max_index; 173 uint8_t *p, *p_end; 174 sigset_t mask; 175 176 _rtld_exclusive_enter(&mask); 177 178 #ifdef __HAVE_TLS_VARIANT_I 179 p = (uint8_t *)tcb; 180 #else 181 p = (uint8_t *)tcb - _rtld_tls_static_space; 182 #endif 183 p_end = p + _rtld_tls_static_space; 184 185 max_index = DTV_MAX_INDEX(tcb->tcb_dtv); 186 for (i = 1; i <= max_index; ++i) { 187 if ((uint8_t *)tcb->tcb_dtv[i] < p || 188 (uint8_t *)tcb->tcb_dtv[i] >= p_end) 189 xfree(tcb->tcb_dtv[i]); 190 } 191 xfree(tcb->tcb_dtv - 1); 192 xfree(p); 193 194 _rtld_exclusive_exit(&mask); 195 } 196 197 void * 198 _rtld_tls_module_allocate(size_t idx) 199 { 200 Obj_Entry *obj; 201 uint8_t *p; 202 203 for (obj = _rtld_objlist; obj != NULL; obj = obj->next) { 204 if (obj->tlsindex == idx) 205 break; 206 } 207 if (obj == NULL) { 208 _rtld_error("Module for TLS index %zu missing", idx); 209 _rtld_die(); 210 } 211 212 p = xmalloc(obj->tlssize); 213 memcpy(p, obj->tlsinit, obj->tlsinitsize); 214 memset(p + obj->tlsinitsize, 0, obj->tlssize - obj->tlsinitsize); 215 216 return p; 217 } 218 219 int 220 _rtld_tls_offset_allocate(Obj_Entry *obj) 221 { 222 size_t offset, next_offset; 223 224 if (obj->tls_done) 225 return 0; 226 if (obj->tlssize == 0) { 227 obj->tlsoffset = 0; 228 obj->tls_done = 1; 229 return 0; 230 } 231 232 #ifdef __HAVE_TLS_VARIANT_I 233 offset = roundup2(_rtld_tls_static_offset, obj->tlsalign); 234 next_offset = offset + obj->tlssize; 235 #else 236 offset = roundup2(_rtld_tls_static_offset + obj->tlssize, 237 obj->tlsalign); 238 next_offset = offset; 239 #endif 240 241 /* 242 * Check if the static allocation was already done. 243 * This happens if dynamically loaded modules want to use 244 * static TLS space. 245 * 246 * XXX Keep an actual free list and callbacks for initialisation. 247 */ 248 if (_rtld_tls_static_space) { 249 if (obj->tlsinitsize) { 250 _rtld_error("%s: Use of initialized " 251 "Thread Local Storage with model initial-exec " 252 "and dlopen is not supported", 253 obj->path); 254 return -1; 255 } 256 if (next_offset > _rtld_tls_static_space) { 257 _rtld_error("%s: No space available " 258 "for static Thread Local Storage", 259 obj->path); 260 return -1; 261 } 262 } 263 obj->tlsoffset = offset; 264 _rtld_tls_static_offset = next_offset; 265 obj->tls_done = 1; 266 267 return 0; 268 } 269 270 void 271 _rtld_tls_offset_free(Obj_Entry *obj) 272 { 273 274 /* 275 * XXX See above. 276 */ 277 obj->tls_done = 0; 278 return; 279 } 280 281 #if defined(__HAVE_COMMON___TLS_GET_ADDR) && defined(RTLD_LOADER) 282 /* 283 * The fast path is access to an already allocated DTV entry. 284 * This checks the current limit and the entry without needing any 285 * locking. Entries are only freed on dlclose() and it is an application 286 * bug if code of the module is still running at that point. 287 */ 288 void * 289 __tls_get_addr(void *arg_) 290 { 291 size_t *arg = (size_t *)arg_; 292 void **dtv; 293 #ifdef __HAVE___LWP_GETTCB_FAST 294 struct tls_tcb * const tcb = __lwp_gettcb_fast(); 295 #else 296 struct tls_tcb * const tcb = __lwp_getprivate_fast(); 297 #endif 298 size_t idx = arg[0], offset = arg[1] + TLS_DTV_OFFSET; 299 300 dtv = tcb->tcb_dtv; 301 302 if (__predict_true(idx < DTV_MAX_INDEX(dtv) && dtv[idx] != NULL)) 303 return (uint8_t *)dtv[idx] + offset; 304 305 return _rtld_tls_get_addr(tcb, idx, offset); 306 } 307 #endif 308 309 #endif /* __HAVE_TLS_VARIANT_I || __HAVE_TLS_VARIANT_II */ 310