5  *  UCRL-CODE-235197
25  *  Thread specific data has implemented using a hash table, this avoids
30  *  The majority of the entries in the hash table are for specific tsd
37  *  By default the hash table is sized to 512 bins which is expected to
40  *  The hash table contains two additional type of entries.  They first
47  *  key you wish to lookup and DTOR_PID constant as the pid.
92  * tsd_hash_search - searches hash table for tsd_hash_entry
93  * @table: hash table
98 tsd_hash_search(tsd_hash_table_t *table, uint_t key, pid_t pid)  in tsd_hash_search()  argument
105 	hash = hash_long((ulong_t)key * (ulong_t)pid, table->ht_bits);  in tsd_hash_search()
106 	bin = &table->ht_bins[hash];  in tsd_hash_search()
107 	spin_lock(&bin->hb_lock);  in tsd_hash_search()
108 	hlist_for_each(node, &bin->hb_head) {  in tsd_hash_search()
110 		if ((entry->he_key == key) && (entry->he_pid == pid)) {  in tsd_hash_search()
111 			spin_unlock(&bin->hb_lock);  in tsd_hash_search()
116 	spin_unlock(&bin->hb_lock);  in tsd_hash_search()
121  * tsd_hash_dtor - call the destructor and free all entries on the list
133 		entry = hlist_entry(work->first, tsd_hash_entry_t, he_list);  in tsd_hash_dtor()
134 		hlist_del(&entry->he_list);  in tsd_hash_dtor()
136 		if (entry->he_dtor && entry->he_pid != DTOR_PID)  in tsd_hash_dtor()
137 			entry->he_dtor(entry->he_value);  in tsd_hash_dtor()
144  * tsd_hash_add - adds an entry to hash table
145  * @table: hash table
150  * already exist in the hash table.  This possible because all entries
153  * links to the dtor and pid entries the entire table is locked.
156 tsd_hash_add(tsd_hash_table_t *table, uint_t key, pid_t pid, void *value)  in tsd_hash_add()  argument
163 	ASSERT3P(tsd_hash_search(table, key, pid), ==, NULL);  in tsd_hash_add()
170 	entry->he_key = key;  in tsd_hash_add()
171 	entry->he_pid = pid;  in tsd_hash_add()
172 	entry->he_value = value;  in tsd_hash_add()
173 	INIT_HLIST_NODE(&entry->he_list);  in tsd_hash_add()
174 	INIT_LIST_HEAD(&entry->he_key_list);  in tsd_hash_add()
175 	INIT_LIST_HEAD(&entry->he_pid_list);  in tsd_hash_add()
177 	spin_lock(&table->ht_lock);  in tsd_hash_add()
180 	dtor_entry = tsd_hash_search(table, entry->he_key, DTOR_PID);  in tsd_hash_add()
182 	entry->he_dtor = dtor_entry->he_dtor;  in tsd_hash_add()
185 	pid_entry = tsd_hash_search(table, PID_KEY, entry->he_pid);  in tsd_hash_add()
188 	hash = hash_long((ulong_t)key * (ulong_t)pid, table->ht_bits);  in tsd_hash_add()
189 	bin = &table->ht_bins[hash];  in tsd_hash_add()
190 	spin_lock(&bin->hb_lock);  in tsd_hash_add()
193 	hlist_add_head(&entry->he_list, &bin->hb_head);  in tsd_hash_add()
194 	list_add(&entry->he_key_list, &dtor_entry->he_key_list);  in tsd_hash_add()
195 	list_add(&entry->he_pid_list, &pid_entry->he_pid_list);  in tsd_hash_add()
197 	spin_unlock(&bin->hb_lock);  in tsd_hash_add()
198 	spin_unlock(&table->ht_lock);  in tsd_hash_add()
204  * tsd_hash_add_key - adds a destructor entry to the hash table
205  * @table: hash table
212  * will be set to the next available key for the hash table.
215 tsd_hash_add_key(tsd_hash_table_t *table, uint_t *keyp, dtor_func_t dtor)  in tsd_hash_add_key()  argument
222 	ASSERT3P(table, !=, NULL);  in tsd_hash_add_key()
230 	spin_lock(&table->ht_lock);  in tsd_hash_add_key()
233 		if (table->ht_key++ > TSD_KEYS_MAX)  in tsd_hash_add_key()
234 			table->ht_key = 1;  in tsd_hash_add_key()
238 			spin_unlock(&table->ht_lock);  in tsd_hash_add_key()
242 		tmp_entry = tsd_hash_search(table, table->ht_key, DTOR_PID);  in tsd_hash_add_key()
245 	/* Add destructor entry in to hash table */  in tsd_hash_add_key()
246 	entry->he_key = *keyp = table->ht_key;  in tsd_hash_add_key()
247 	entry->he_pid = DTOR_PID;  in tsd_hash_add_key()
248 	entry->he_dtor = dtor;  in tsd_hash_add_key()
249 	entry->he_value = NULL;  in tsd_hash_add_key()
250 	INIT_HLIST_NODE(&entry->he_list);  in tsd_hash_add_key()
251 	INIT_LIST_HEAD(&entry->he_key_list);  in tsd_hash_add_key()
252 	INIT_LIST_HEAD(&entry->he_pid_list);  in tsd_hash_add_key()
254 	hash = hash_long((ulong_t)*keyp * (ulong_t)DTOR_PID, table->ht_bits);  in tsd_hash_add_key()
255 	bin = &table->ht_bins[hash];  in tsd_hash_add_key()
256 	spin_lock(&bin->hb_lock);  in tsd_hash_add_key()
258 	hlist_add_head(&entry->he_list, &bin->hb_head);  in tsd_hash_add_key()
260 	spin_unlock(&bin->hb_lock);  in tsd_hash_add_key()
261 	spin_unlock(&table->ht_lock);  in tsd_hash_add_key()
267  * tsd_hash_add_pid - adds a process entry to the hash table
268  * @table: hash table
276 tsd_hash_add_pid(tsd_hash_table_t *table, pid_t pid)  in tsd_hash_add_pid()  argument
287 	spin_lock(&table->ht_lock);  in tsd_hash_add_pid()
288 	entry->he_key = PID_KEY;  in tsd_hash_add_pid()
289 	entry->he_pid = pid;  in tsd_hash_add_pid()
290 	entry->he_dtor = NULL;  in tsd_hash_add_pid()
291 	entry->he_value = NULL;  in tsd_hash_add_pid()
292 	INIT_HLIST_NODE(&entry->he_list);  in tsd_hash_add_pid()
293 	INIT_LIST_HEAD(&entry->he_key_list);  in tsd_hash_add_pid()
294 	INIT_LIST_HEAD(&entry->he_pid_list);  in tsd_hash_add_pid()
296 	hash = hash_long((ulong_t)PID_KEY * (ulong_t)pid, table->ht_bits);  in tsd_hash_add_pid()
297 	bin = &table->ht_bins[hash];  in tsd_hash_add_pid()
298 	spin_lock(&bin->hb_lock);  in tsd_hash_add_pid()
300 	hlist_add_head(&entry->he_list, &bin->hb_head);  in tsd_hash_add_pid()
302 	spin_unlock(&bin->hb_lock);  in tsd_hash_add_pid()
303 	spin_unlock(&table->ht_lock);  in tsd_hash_add_pid()
309  * tsd_hash_del - delete an entry from hash table, key, and pid lists
310  * @table: hash table
315 tsd_hash_del(tsd_hash_table_t *table, tsd_hash_entry_t *entry)  in tsd_hash_del()  argument
317 	hlist_del(&entry->he_list);  in tsd_hash_del()
318 	list_del_init(&entry->he_key_list);  in tsd_hash_del()
319 	list_del_init(&entry->he_pid_list);  in tsd_hash_del()
323  * tsd_hash_table_init - allocate a hash table
324  * @bits: hash table size
326  * A hash table with 2^bits bins will be created, it may not be resized
332 	tsd_hash_table_t *table;  in tsd_hash_table_init()  local
335 	table = kmem_zalloc(sizeof (tsd_hash_table_t), KM_SLEEP);  in tsd_hash_table_init()
336 	if (table == NULL)  in tsd_hash_table_init()
339 	table->ht_bins = kmem_zalloc(sizeof (tsd_hash_bin_t) * size, KM_SLEEP);  in tsd_hash_table_init()
340 	if (table->ht_bins == NULL) {  in tsd_hash_table_init()
341 		kmem_free(table, sizeof (tsd_hash_table_t));  in tsd_hash_table_init()
346 		spin_lock_init(&table->ht_bins[hash].hb_lock);  in tsd_hash_table_init()
347 		INIT_HLIST_HEAD(&table->ht_bins[hash].hb_head);  in tsd_hash_table_init()
350 	spin_lock_init(&table->ht_lock);  in tsd_hash_table_init()
351 	table->ht_bits = bits;  in tsd_hash_table_init()
352 	table->ht_key = 1;  in tsd_hash_table_init()
354 	return (table);  in tsd_hash_table_init()
358  * tsd_hash_table_fini - free a hash table
359  * @table: hash table
361  * Free a hash table allocated by tsd_hash_table_init().  If the hash
362  * table is not empty this function will call the proper destructor for
366 tsd_hash_table_fini(tsd_hash_table_t *table)  in tsd_hash_table_fini()  argument
373 	ASSERT3P(table, !=, NULL);  in tsd_hash_table_fini()
374 	spin_lock(&table->ht_lock);  in tsd_hash_table_fini()
375 	for (i = 0, size = (1 << table->ht_bits); i < size; i++) {  in tsd_hash_table_fini()
376 		bin = &table->ht_bins[i];  in tsd_hash_table_fini()
377 		spin_lock(&bin->hb_lock);  in tsd_hash_table_fini()
378 		while (!hlist_empty(&bin->hb_head)) {  in tsd_hash_table_fini()
379 			entry = hlist_entry(bin->hb_head.first,  in tsd_hash_table_fini()
381 			tsd_hash_del(table, entry);  in tsd_hash_table_fini()
382 			hlist_add_head(&entry->he_list, &work);  in tsd_hash_table_fini()
384 		spin_unlock(&bin->hb_lock);  in tsd_hash_table_fini()
386 	spin_unlock(&table->ht_lock);  in tsd_hash_table_fini()
389 	kmem_free(table->ht_bins, sizeof (tsd_hash_bin_t)*(1<<table->ht_bits));  in tsd_hash_table_fini()
390 	kmem_free(table, sizeof (tsd_hash_table_t));  in tsd_hash_table_fini()
394  * tsd_remove_entry - remove a tsd entry for this thread
404 	tsd_hash_table_t *table;  in tsd_remove_entry()  local
409 	table = tsd_hash_table;  in tsd_remove_entry()
410 	ASSERT3P(table, !=, NULL);  in tsd_remove_entry()
413 	spin_lock(&table->ht_lock);  in tsd_remove_entry()
415 	hash = hash_long((ulong_t)entry->he_key *  in tsd_remove_entry()
416 	    (ulong_t)entry->he_pid, table->ht_bits);  in tsd_remove_entry()
417 	entry_bin = &table->ht_bins[hash];  in tsd_remove_entry()
420 	pid_entry = list_entry(entry->he_pid_list.next, tsd_hash_entry_t,  in tsd_remove_entry()
424 	spin_lock(&entry_bin->hb_lock);  in tsd_remove_entry()
425 	tsd_hash_del(table, entry);  in tsd_remove_entry()
426 	hlist_add_head(&entry->he_list, &work);  in tsd_remove_entry()
427 	spin_unlock(&entry_bin->hb_lock);  in tsd_remove_entry()
430 	if (pid_entry->he_key == PID_KEY &&  in tsd_remove_entry()
431 	    list_empty(&pid_entry->he_pid_list)) {  in tsd_remove_entry()
432 		hash = hash_long((ulong_t)pid_entry->he_key *  in tsd_remove_entry()
433 		    (ulong_t)pid_entry->he_pid, table->ht_bits);  in tsd_remove_entry()
434 		pid_entry_bin = &table->ht_bins[hash];  in tsd_remove_entry()
436 		spin_lock(&pid_entry_bin->hb_lock);  in tsd_remove_entry()
437 		tsd_hash_del(table, pid_entry);  in tsd_remove_entry()
438 		hlist_add_head(&pid_entry->he_list, &work);  in tsd_remove_entry()
439 		spin_unlock(&pid_entry_bin->hb_lock);  in tsd_remove_entry()
442 	spin_unlock(&table->ht_lock);  in tsd_remove_entry()
448  * tsd_set - set thread specific data
449  * @key: lookup key
461 	tsd_hash_table_t *table;  in tsd_set()  local
468 	table = tsd_hash_table;  in tsd_set()
469 	pid = curthread->pid;  in tsd_set()
470 	ASSERT3P(table, !=, NULL);  in tsd_set()
475 	/* Entry already exists in hash table update value */  in tsd_set()
476 	entry = tsd_hash_search(table, key, pid);  in tsd_set()
478 		entry->he_value = value;  in tsd_set()
490 	entry = tsd_hash_search(table, PID_KEY, pid);  in tsd_set()
492 		rc = tsd_hash_add_pid(table, pid);  in tsd_set()
497 	rc = tsd_hash_add(table, key, pid, value);  in tsd_set()
503  * tsd_get - get thread specific data
504  * @key: lookup key
508  * lock the entire table only a single hash bin.
520 	entry = tsd_hash_search(tsd_hash_table, key, curthread->pid);  in tsd_get()
524 	return (entry->he_value);  in tsd_get()
529  * tsd_get_by_thread - get thread specific data for specified thread
530  * @key: lookup key
531  * @thread: thread to lookup
535  * lock the entire table only a single hash bin.
547 	entry = tsd_hash_search(tsd_hash_table, key, thread->pid);  in tsd_get_by_thread()
551 	return (entry->he_value);  in tsd_get_by_thread()
556  * tsd_create - create thread specific data key
557  * @keyp: lookup key address
579  * tsd_destroy - destroy thread specific data
580  * @keyp: lookup key address
591 	tsd_hash_table_t *table;  in tsd_destroy()  local
596 	table = tsd_hash_table;  in tsd_destroy()
597 	ASSERT3P(table, !=, NULL);  in tsd_destroy()
599 	spin_lock(&table->ht_lock);  in tsd_destroy()
600 	dtor_entry = tsd_hash_search(table, *keyp, DTOR_PID);  in tsd_destroy()
602 		spin_unlock(&table->ht_lock);  in tsd_destroy()
608 	 * DTOR_PID entry.  They are removed from the hash table and  in tsd_destroy()
611 	while (!list_empty(&dtor_entry->he_key_list)) {  in tsd_destroy()
612 		entry = list_entry(dtor_entry->he_key_list.next,  in tsd_destroy()
614 		ASSERT3U(dtor_entry->he_key, ==, entry->he_key);  in tsd_destroy()
615 		ASSERT3P(dtor_entry->he_dtor, ==, entry->he_dtor);  in tsd_destroy()
617 		hash = hash_long((ulong_t)entry->he_key *  in tsd_destroy()
618 		    (ulong_t)entry->he_pid, table->ht_bits);  in tsd_destroy()
619 		entry_bin = &table->ht_bins[hash];  in tsd_destroy()
621 		spin_lock(&entry_bin->hb_lock);  in tsd_destroy()
622 		tsd_hash_del(table, entry);  in tsd_destroy()
623 		hlist_add_head(&entry->he_list, &work);  in tsd_destroy()
624 		spin_unlock(&entry_bin->hb_lock);  in tsd_destroy()
627 	hash = hash_long((ulong_t)dtor_entry->he_key *  in tsd_destroy()
628 	    (ulong_t)dtor_entry->he_pid, table->ht_bits);  in tsd_destroy()
629 	dtor_entry_bin = &table->ht_bins[hash];  in tsd_destroy()
631 	spin_lock(&dtor_entry_bin->hb_lock);  in tsd_destroy()
632 	tsd_hash_del(table, dtor_entry);  in tsd_destroy()
633 	hlist_add_head(&dtor_entry->he_list, &work);  in tsd_destroy()
634 	spin_unlock(&dtor_entry_bin->hb_lock);  in tsd_destroy()
635 	spin_unlock(&table->ht_lock);  in tsd_destroy()
643  * tsd_exit - destroys all thread specific data for this thread
654 	tsd_hash_table_t *table;  in tsd_exit()  local
659 	table = tsd_hash_table;  in tsd_exit()
660 	ASSERT3P(table, !=, NULL);  in tsd_exit()
662 	spin_lock(&table->ht_lock);  in tsd_exit()
663 	pid_entry = tsd_hash_search(table, PID_KEY, curthread->pid);  in tsd_exit()
665 		spin_unlock(&table->ht_lock);  in tsd_exit()
671 	 * PID_KEY entry.  They are removed from the hash table and  in tsd_exit()
675 	while (!list_empty(&pid_entry->he_pid_list)) {  in tsd_exit()
676 		entry = list_entry(pid_entry->he_pid_list.next,  in tsd_exit()
678 		ASSERT3U(pid_entry->he_pid, ==, entry->he_pid);  in tsd_exit()
680 		hash = hash_long((ulong_t)entry->he_key *  in tsd_exit()
681 		    (ulong_t)entry->he_pid, table->ht_bits);  in tsd_exit()
682 		entry_bin = &table->ht_bins[hash];  in tsd_exit()
684 		spin_lock(&entry_bin->hb_lock);  in tsd_exit()
685 		tsd_hash_del(table, entry);  in tsd_exit()
686 		hlist_add_head(&entry->he_list, &work);  in tsd_exit()
687 		spin_unlock(&entry_bin->hb_lock);  in tsd_exit()
690 	hash = hash_long((ulong_t)pid_entry->he_key *  in tsd_exit()
691 	    (ulong_t)pid_entry->he_pid, table->ht_bits);  in tsd_exit()
692 	pid_entry_bin = &table->ht_bins[hash];  in tsd_exit()
694 	spin_lock(&pid_entry_bin->hb_lock);  in tsd_exit()
695 	tsd_hash_del(table, pid_entry);  in tsd_exit()
696 	hlist_add_head(&pid_entry->he_list, &work);  in tsd_exit()
697 	spin_unlock(&pid_entry_bin->hb_lock);  in tsd_exit()
698 	spin_unlock(&table->ht_lock);  in tsd_exit()
709 		return (-ENOMEM);  in spl_tsd_init()