132 static int sa_build_index(sa_handle_t *hdl, sa_buf_type_t buftype);
139 static int sa_modify_attrs(sa_handle_t *hdl, sa_attr_type_t newattr,
213 	sa_handle_t *hdl = buf;  in sa_cache_constructor()  local
215 	mutex_init(&hdl->sa_lock, NULL, MUTEX_DEFAULT, NULL);  in sa_cache_constructor()
223 	sa_handle_t *hdl = buf;  in sa_cache_destructor()  local
225 	mutex_destroy(&hdl->sa_lock);  in sa_cache_destructor()
303 sa_get_spill(sa_handle_t *hdl)  in sa_get_spill()  argument
306 	if (hdl->sa_spill == NULL) {  in sa_get_spill()
307 		if ((rc = dmu_spill_hold_existing(hdl->sa_bonus, NULL,  in sa_get_spill()
308 		    &hdl->sa_spill)) == 0)  in sa_get_spill()
309 			VERIFY(0 == sa_build_index(hdl, SA_SPILL));  in sa_get_spill()
324 sa_attr_op(sa_handle_t *hdl, sa_bulk_attr_t *bulk, int count,  in sa_attr_op()  argument
327 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_attr_op()
336 		ASSERT(bulk[i].sa_attr <= hdl->sa_os->os_sa->sa_num_attrs);  in sa_attr_op()
341 		if (hdl->sa_bonus_tab && TOC_ATTR_PRESENT(  in sa_attr_op()
342 		    hdl->sa_bonus_tab->sa_idx_tab[bulk[i].sa_attr])) {  in sa_attr_op()
343 			SA_ATTR_INFO(sa, hdl->sa_bonus_tab,  in sa_attr_op()
344 			    SA_GET_HDR(hdl, SA_BONUS),  in sa_attr_op()
345 			    bulk[i].sa_attr, bulk[i], SA_BONUS, hdl);  in sa_attr_op()
347 				dmu_buf_will_dirty(hdl->sa_bonus, tx);  in sa_attr_op()
352 		    ((error = sa_get_spill(hdl)) == 0)) {  in sa_attr_op()
354 			    hdl->sa_spill_tab->sa_idx_tab[bulk[i].sa_attr])) {  in sa_attr_op()
355 				SA_ATTR_INFO(sa, hdl->sa_spill_tab,  in sa_attr_op()
356 				    SA_GET_HDR(hdl, SA_SPILL),  in sa_attr_op()
357 				    bulk[i].sa_attr, bulk[i], SA_SPILL, hdl);  in sa_attr_op()
360 					dmu_buf_will_dirty(hdl->sa_spill, tx);  in sa_attr_op()
389 				error = sa_modify_attrs(hdl, bulk[i].sa_attr,  in sa_attr_op()
393 				error = sa_modify_attrs(hdl, bulk[i].sa_attr,  in sa_attr_op()
496 sa_resize_spill(sa_handle_t *hdl, uint32_t size, dmu_tx_t *tx)  in sa_resize_spill()  argument
510 	error = dbuf_spill_set_blksz(hdl->sa_spill, blocksize, tx);  in sa_resize_spill()
649 sa_build_layouts(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc, int attr_count,  in sa_build_layouts()  argument
652 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_build_layouts()
669 	dmu_buf_will_dirty(hdl->sa_bonus, tx);  in sa_build_layouts()
670 	bonustype = SA_BONUSTYPE_FROM_DB(hdl->sa_bonus);  in sa_build_layouts()
673 	hdrsize = sa_find_sizes(sa, attr_desc, attr_count, hdl->sa_bonus,  in sa_build_layouts()
679 	VERIFY(0 == dmu_set_bonus(hdl->sa_bonus, spilling ?  in sa_build_layouts()
690 		if (hdl->sa_spill == NULL) {  in sa_build_layouts()
691 			VERIFY(dmu_spill_hold_by_bonus(hdl->sa_bonus, NULL,  in sa_build_layouts()
692 			    &hdl->sa_spill) == 0);  in sa_build_layouts()
694 		dmu_buf_will_dirty(hdl->sa_spill, tx);  in sa_build_layouts()
697 		    attr_count - i, hdl->sa_spill, SA_SPILL, &i,  in sa_build_layouts()
703 		buf_space = hdl->sa_spill->db_size - spillhdrsize;  in sa_build_layouts()
705 		    hdl->sa_spill->db_size)  in sa_build_layouts()
706 			VERIFY(0 == sa_resize_spill(hdl,  in sa_build_layouts()
711 	data_start = (void *)((uintptr_t)hdl->sa_bonus->db_data + hdrsize);  in sa_build_layouts()
712 	sahdr = (sa_hdr_phys_t *)hdl->sa_bonus->db_data;  in sa_build_layouts()
719 		buf_space = hdl->sa_bonus->db_size - hdrsize;  in sa_build_layouts()
741 				sa_find_layout(hdl->sa_os, hash, attrs_start,  in sa_build_layouts()
750 			sahdr = (sa_hdr_phys_t *)hdl->sa_spill->db_data;  in sa_build_layouts()
755 			buf_space = hdl->sa_spill->db_size - spillhdrsize;  in sa_build_layouts()
773 	sa_find_layout(hdl->sa_os, hash, attrs_start, lot_count, tx, &lot);  in sa_build_layouts()
788 	if (hdl->sa_bonus_tab) {  in sa_build_layouts()
789 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);  in sa_build_layouts()
790 		hdl->sa_bonus_tab = NULL;  in sa_build_layouts()
793 		VERIFY(0 == sa_build_index(hdl, SA_BONUS));  in sa_build_layouts()
794 	if (hdl->sa_spill) {  in sa_build_layouts()
795 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);  in sa_build_layouts()
800 			dmu_buf_rele(hdl->sa_spill, NULL);  in sa_build_layouts()
801 			hdl->sa_spill = NULL;  in sa_build_layouts()
802 			hdl->sa_spill_tab = NULL;  in sa_build_layouts()
803 			VERIFY(0 == dmu_rm_spill(hdl->sa_os,  in sa_build_layouts()
804 			    sa_handle_object(hdl), tx));  in sa_build_layouts()
806 			VERIFY(0 == sa_build_index(hdl, SA_SPILL));  in sa_build_layouts()
1227 	sa_handle_t *hdl = userp;  in sa_byteswap_cb()  local
1228 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_byteswap_cb()
1234 sa_byteswap(sa_handle_t *hdl, sa_buf_type_t buftype)  in sa_byteswap()  argument
1236 	sa_hdr_phys_t *sa_hdr_phys = SA_GET_HDR(hdl, buftype);  in sa_byteswap()
1238 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_byteswap()
1246 	db = SA_GET_DB(hdl, buftype);  in sa_byteswap()
1267 	sa_attr_iter(hdl->sa_os, sa_hdr_phys, DMU_OT_SA,  in sa_byteswap()
1268 	    sa_byteswap_cb, NULL, hdl);  in sa_byteswap()
1271 		arc_buf_freeze(((dmu_buf_impl_t *)hdl->sa_spill)->db_buf);  in sa_byteswap()
1275 sa_build_index(sa_handle_t *hdl, sa_buf_type_t buftype)  in sa_build_index()  argument
1278 	dmu_buf_impl_t *db = SA_GET_DB(hdl, buftype);  in sa_build_index()
1280 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_build_index()
1283 	sa_hdr_phys = SA_GET_HDR(hdl, buftype);  in sa_build_index()
1293 		sa_byteswap(hdl, buftype);  in sa_build_index()
1296 	idx_tab = sa_find_idx_tab(hdl->sa_os, bonustype, sa_hdr_phys);  in sa_build_index()
1299 		hdl->sa_bonus_tab = idx_tab;  in sa_build_index()
1301 		hdl->sa_spill_tab = idx_tab;  in sa_build_index()
1348 sa_handle_destroy(sa_handle_t *hdl)  in sa_handle_destroy()  argument
1350 	dmu_buf_t *db = hdl->sa_bonus;  in sa_handle_destroy()
1352 	mutex_enter(&hdl->sa_lock);  in sa_handle_destroy()
1353 	(void) dmu_buf_remove_user(db, &hdl->sa_dbu);  in sa_handle_destroy()
1355 	if (hdl->sa_bonus_tab)  in sa_handle_destroy()
1356 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);  in sa_handle_destroy()
1358 	if (hdl->sa_spill_tab)  in sa_handle_destroy()
1359 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_spill_tab);  in sa_handle_destroy()
1361 	dmu_buf_rele(hdl->sa_bonus, NULL);  in sa_handle_destroy()
1363 	if (hdl->sa_spill)  in sa_handle_destroy()
1364 		dmu_buf_rele((dmu_buf_t *)hdl->sa_spill, NULL);  in sa_handle_destroy()
1365 	mutex_exit(&hdl->sa_lock);  in sa_handle_destroy()
1367 	kmem_cache_free(sa_cache, hdl);  in sa_handle_destroy()
1447 sa_lookup_impl(sa_handle_t *hdl, sa_bulk_attr_t *bulk, int count)  in sa_lookup_impl()  argument
1449 	ASSERT(hdl);  in sa_lookup_impl()
1450 	ASSERT(MUTEX_HELD(&hdl->sa_lock));  in sa_lookup_impl()
1451 	return (sa_attr_op(hdl, bulk, count, SA_LOOKUP, NULL));  in sa_lookup_impl()
1455 sa_lookup(sa_handle_t *hdl, sa_attr_type_t attr, void *buf, uint32_t buflen)  in sa_lookup()  argument
1465 	ASSERT(hdl);  in sa_lookup()
1466 	mutex_enter(&hdl->sa_lock);  in sa_lookup()
1467 	error = sa_lookup_impl(hdl, &bulk, 1);  in sa_lookup()
1468 	mutex_exit(&hdl->sa_lock);  in sa_lookup()
1474 sa_lookup_uio(sa_handle_t *hdl, sa_attr_type_t attr, uio_t *uio)  in sa_lookup_uio()  argument
1483 	ASSERT(hdl);  in sa_lookup_uio()
1485 	mutex_enter(&hdl->sa_lock);  in sa_lookup_uio()
1486 	if ((error = sa_attr_op(hdl, &bulk, 1, SA_LOOKUP, NULL)) == 0) {  in sa_lookup_uio()
1490 	mutex_exit(&hdl->sa_lock);  in sa_lookup_uio()
1578 sa_attr_register_sync(sa_handle_t *hdl, dmu_tx_t *tx)  in sa_attr_register_sync()  argument
1581 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_attr_register_sync()
1593 		sa->sa_reg_attr_obj = zap_create_link(hdl->sa_os,  in sa_attr_register_sync()
1602 		VERIFY(0 == zap_update(hdl->sa_os, sa->sa_reg_attr_obj,  in sa_attr_register_sync()
1620 sa_replace_all_by_template_locked(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc,  in sa_replace_all_by_template_locked()  argument
1623 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_replace_all_by_template_locked()
1626 		sa_attr_register_sync(hdl, tx);  in sa_replace_all_by_template_locked()
1627 	return (sa_build_layouts(hdl, attr_desc, attr_count, tx));  in sa_replace_all_by_template_locked()
1631 sa_replace_all_by_template(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc,  in sa_replace_all_by_template()  argument
1636 	mutex_enter(&hdl->sa_lock);  in sa_replace_all_by_template()
1637 	error = sa_replace_all_by_template_locked(hdl, attr_desc,  in sa_replace_all_by_template()
1639 	mutex_exit(&hdl->sa_lock);  in sa_replace_all_by_template()
1651 sa_modify_attrs(sa_handle_t *hdl, sa_attr_type_t newattr,  in sa_modify_attrs()  argument
1655 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_modify_attrs()
1656 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)hdl->sa_bonus;  in sa_modify_attrs()
1672 	ASSERT(MUTEX_HELD(&hdl->sa_lock));  in sa_modify_attrs()
1679 		bonus_data_size = hdl->sa_bonus->db_size;  in sa_modify_attrs()
1681 		bcopy(hdl->sa_bonus->db_data, old_data[0],  in sa_modify_attrs()
1682 		    hdl->sa_bonus->db_size);  in sa_modify_attrs()
1683 		bonus_attr_count = hdl->sa_bonus_tab->sa_layout->lot_attr_count;  in sa_modify_attrs()
1691 	if ((error = sa_get_spill(hdl)) == 0) {  in sa_modify_attrs()
1692 		spill_data_size = hdl->sa_spill->db_size;  in sa_modify_attrs()
1694 		bcopy(hdl->sa_spill->db_data, old_data[1],  in sa_modify_attrs()
1695 		    hdl->sa_spill->db_size);  in sa_modify_attrs()
1697 		    hdl->sa_spill_tab->sa_layout->lot_attr_count;  in sa_modify_attrs()
1722 	hdr = SA_GET_HDR(hdl, SA_BONUS);  in sa_modify_attrs()
1723 	idx_tab = SA_IDX_TAB_GET(hdl, SA_BONUS);  in sa_modify_attrs()
1770 		if (k == 0 && hdl->sa_spill) {  in sa_modify_attrs()
1771 			hdr = SA_GET_HDR(hdl, SA_SPILL);  in sa_modify_attrs()
1772 			idx_tab = SA_IDX_TAB_GET(hdl, SA_SPILL);  in sa_modify_attrs()
1786 	error = sa_build_layouts(hdl, attr_desc, attr_count, tx);  in sa_modify_attrs()
1798 sa_bulk_update_impl(sa_handle_t *hdl, sa_bulk_attr_t *bulk, int count,  in sa_bulk_update_impl()  argument
1802 	sa_os_t *sa = hdl->sa_os->os_sa;  in sa_bulk_update_impl()
1805 	bonustype = SA_BONUSTYPE_FROM_DB(SA_GET_DB(hdl, SA_BONUS));  in sa_bulk_update_impl()
1807 	ASSERT(hdl);  in sa_bulk_update_impl()
1808 	ASSERT(MUTEX_HELD(&hdl->sa_lock));  in sa_bulk_update_impl()
1812 		sa_attr_register_sync(hdl, tx);  in sa_bulk_update_impl()
1814 	error = sa_attr_op(hdl, bulk, count, SA_UPDATE, tx);  in sa_bulk_update_impl()
1816 		sa->sa_update_cb(hdl, tx);  in sa_bulk_update_impl()
1825 sa_update(sa_handle_t *hdl, sa_attr_type_t type,  in sa_update()  argument
1836 	mutex_enter(&hdl->sa_lock);  in sa_update()
1837 	error = sa_bulk_update_impl(hdl, &bulk, 1, tx);  in sa_update()
1838 	mutex_exit(&hdl->sa_lock);  in sa_update()
1843 sa_update_from_cb(sa_handle_t *hdl, sa_attr_type_t attr,  in sa_update_from_cb()  argument
1854 	mutex_enter(&hdl->sa_lock);  in sa_update_from_cb()
1855 	error = sa_bulk_update_impl(hdl, &bulk, 1, tx);  in sa_update_from_cb()
1856 	mutex_exit(&hdl->sa_lock);  in sa_update_from_cb()
1865 sa_size(sa_handle_t *hdl, sa_attr_type_t attr, int *size)  in sa_size()  argument
1874 	ASSERT(hdl);  in sa_size()
1875 	mutex_enter(&hdl->sa_lock);  in sa_size()
1876 	if ((error = sa_attr_op(hdl, &bulk, 1, SA_LOOKUP, NULL)) != 0) {  in sa_size()
1877 		mutex_exit(&hdl->sa_lock);  in sa_size()
1882 	mutex_exit(&hdl->sa_lock);  in sa_size()
1887 sa_bulk_lookup_locked(sa_handle_t *hdl, sa_bulk_attr_t *attrs, int count)  in sa_bulk_lookup_locked()  argument
1889 	ASSERT(hdl);  in sa_bulk_lookup_locked()
1890 	ASSERT(MUTEX_HELD(&hdl->sa_lock));  in sa_bulk_lookup_locked()
1891 	return (sa_lookup_impl(hdl, attrs, count));  in sa_bulk_lookup_locked()
1895 sa_bulk_lookup(sa_handle_t *hdl, sa_bulk_attr_t *attrs, int count)  in sa_bulk_lookup()  argument
1899 	ASSERT(hdl);  in sa_bulk_lookup()
1900 	mutex_enter(&hdl->sa_lock);  in sa_bulk_lookup()
1901 	error = sa_bulk_lookup_locked(hdl, attrs, count);  in sa_bulk_lookup()
1902 	mutex_exit(&hdl->sa_lock);  in sa_bulk_lookup()
1907 sa_bulk_update(sa_handle_t *hdl, sa_bulk_attr_t *attrs, int count, dmu_tx_t *tx)  in sa_bulk_update()  argument
1911 	ASSERT(hdl);  in sa_bulk_update()
1912 	mutex_enter(&hdl->sa_lock);  in sa_bulk_update()
1913 	error = sa_bulk_update_impl(hdl, attrs, count, tx);  in sa_bulk_update()
1914 	mutex_exit(&hdl->sa_lock);  in sa_bulk_update()
1919 sa_remove(sa_handle_t *hdl, sa_attr_type_t attr, dmu_tx_t *tx)  in sa_remove()  argument
1923 	mutex_enter(&hdl->sa_lock);  in sa_remove()
1924 	error = sa_modify_attrs(hdl, attr, SA_REMOVE, NULL,  in sa_remove()
1926 	mutex_exit(&hdl->sa_lock);  in sa_remove()
1931 sa_object_info(sa_handle_t *hdl, dmu_object_info_t *doi)  in sa_object_info()  argument
1933 	dmu_object_info_from_db((dmu_buf_t *)hdl->sa_bonus, doi);  in sa_object_info()
1937 sa_object_size(sa_handle_t *hdl, uint32_t *blksize, u_longlong_t *nblocks)  in sa_object_size()  argument
1939 	dmu_object_size_from_db((dmu_buf_t *)hdl->sa_bonus,  in sa_object_size()
1944 sa_set_userp(sa_handle_t *hdl, void *ptr)  in sa_set_userp()  argument
1946 	hdl->sa_userp = ptr;  in sa_set_userp()
1950 sa_get_db(sa_handle_t *hdl)  in sa_get_db()  argument
1952 	return ((dmu_buf_t *)hdl->sa_bonus);  in sa_get_db()
1956 sa_get_userdata(sa_handle_t *hdl)  in sa_get_userdata()  argument
1958 	return (hdl->sa_userp);  in sa_get_userdata()
1978 sa_handle_object(sa_handle_t *hdl)  in sa_handle_object()  argument
1980 	return (hdl->sa_bonus->db_object);  in sa_handle_object()
2011 sa_handle_lock(sa_handle_t *hdl)  in sa_handle_lock()  argument
2013 	ASSERT(hdl);  in sa_handle_lock()
2014 	mutex_enter(&hdl->sa_lock);  in sa_handle_lock()
2018 sa_handle_unlock(sa_handle_t *hdl)  in sa_handle_unlock()  argument
2020 	ASSERT(hdl);  in sa_handle_unlock()
2021 	mutex_exit(&hdl->sa_lock);  in sa_handle_unlock()