1eda14cbcSMatt Macy /*
2eda14cbcSMatt Macy * CDDL HEADER START
3eda14cbcSMatt Macy *
4eda14cbcSMatt Macy * The contents of this file are subject to the terms of the
5eda14cbcSMatt Macy * Common Development and Distribution License (the "License").
6eda14cbcSMatt Macy * You may not use this file except in compliance with the License.
7eda14cbcSMatt Macy *
8eda14cbcSMatt Macy * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9*271171e0SMartin Matuska * or https://opensource.org/licenses/CDDL-1.0.
10eda14cbcSMatt Macy * See the License for the specific language governing permissions
11eda14cbcSMatt Macy * and limitations under the License.
12eda14cbcSMatt Macy *
13eda14cbcSMatt Macy * When distributing Covered Code, include this CDDL HEADER in each
14eda14cbcSMatt Macy * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15eda14cbcSMatt Macy * If applicable, add the following below this CDDL HEADER, with the
16eda14cbcSMatt Macy * fields enclosed by brackets "[]" replaced with your own identifying
17eda14cbcSMatt Macy * information: Portions Copyright [yyyy] [name of copyright owner]
18eda14cbcSMatt Macy *
19eda14cbcSMatt Macy * CDDL HEADER END
20eda14cbcSMatt Macy */
21eda14cbcSMatt Macy /*
22eda14cbcSMatt Macy * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23eda14cbcSMatt Macy * Use is subject to license terms.
24eda14cbcSMatt Macy */
25eda14cbcSMatt Macy /*
26eda14cbcSMatt Macy * Copyright (c) 2012, 2019 by Delphix. All rights reserved.
27eda14cbcSMatt Macy */
28eda14cbcSMatt Macy
29eda14cbcSMatt Macy #include <sys/zfs_context.h>
30eda14cbcSMatt Macy #include <sys/spa.h>
31eda14cbcSMatt Macy #include <sys/dmu.h>
32eda14cbcSMatt Macy #include <sys/dmu_tx.h>
33eda14cbcSMatt Macy #include <sys/dnode.h>
34eda14cbcSMatt Macy #include <sys/dsl_pool.h>
35eda14cbcSMatt Macy #include <sys/zio.h>
36eda14cbcSMatt Macy #include <sys/space_map.h>
37eda14cbcSMatt Macy #include <sys/zfeature.h>
38eda14cbcSMatt Macy
39eda14cbcSMatt Macy /*
40eda14cbcSMatt Macy * Note on space map block size:
41eda14cbcSMatt Macy *
42eda14cbcSMatt Macy * The data for a given space map can be kept on blocks of any size.
43eda14cbcSMatt Macy * Larger blocks entail fewer I/O operations, but they also cause the
44eda14cbcSMatt Macy * DMU to keep more data in-core, and also to waste more I/O bandwidth
45eda14cbcSMatt Macy * when only a few blocks have changed since the last transaction group.
46eda14cbcSMatt Macy */
47eda14cbcSMatt Macy
48eda14cbcSMatt Macy /*
49eda14cbcSMatt Macy * Enabled whenever we want to stress test the use of double-word
50eda14cbcSMatt Macy * space map entries.
51eda14cbcSMatt Macy */
52eda14cbcSMatt Macy boolean_t zfs_force_some_double_word_sm_entries = B_FALSE;
53eda14cbcSMatt Macy
54eda14cbcSMatt Macy /*
55eda14cbcSMatt Macy * Override the default indirect block size of 128K, instead use 16K for
56eda14cbcSMatt Macy * spacemaps (2^14 bytes). This dramatically reduces write inflation since
57eda14cbcSMatt Macy * appending to a spacemap typically has to write one data block (4KB) and one
58eda14cbcSMatt Macy * or two indirect blocks (16K-32K, rather than 128K).
59eda14cbcSMatt Macy */
60eda14cbcSMatt Macy int space_map_ibs = 14;
61eda14cbcSMatt Macy
62eda14cbcSMatt Macy boolean_t
sm_entry_is_debug(uint64_t e)63eda14cbcSMatt Macy sm_entry_is_debug(uint64_t e)
64eda14cbcSMatt Macy {
65eda14cbcSMatt Macy return (SM_PREFIX_DECODE(e) == SM_DEBUG_PREFIX);
66eda14cbcSMatt Macy }
67eda14cbcSMatt Macy
68eda14cbcSMatt Macy boolean_t
sm_entry_is_single_word(uint64_t e)69eda14cbcSMatt Macy sm_entry_is_single_word(uint64_t e)
70eda14cbcSMatt Macy {
71eda14cbcSMatt Macy uint8_t prefix = SM_PREFIX_DECODE(e);
72eda14cbcSMatt Macy return (prefix != SM_DEBUG_PREFIX && prefix != SM2_PREFIX);
73eda14cbcSMatt Macy }
74eda14cbcSMatt Macy
75eda14cbcSMatt Macy boolean_t
sm_entry_is_double_word(uint64_t e)76eda14cbcSMatt Macy sm_entry_is_double_word(uint64_t e)
77eda14cbcSMatt Macy {
78eda14cbcSMatt Macy return (SM_PREFIX_DECODE(e) == SM2_PREFIX);
79eda14cbcSMatt Macy }
80eda14cbcSMatt Macy
81eda14cbcSMatt Macy /*
82eda14cbcSMatt Macy * Iterate through the space map, invoking the callback on each (non-debug)
83eda14cbcSMatt Macy * space map entry. Stop after reading 'end' bytes of the space map.
84eda14cbcSMatt Macy */
85eda14cbcSMatt Macy int
space_map_iterate(space_map_t * sm,uint64_t end,sm_cb_t callback,void * arg)86eda14cbcSMatt Macy space_map_iterate(space_map_t *sm, uint64_t end, sm_cb_t callback, void *arg)
87eda14cbcSMatt Macy {
88eda14cbcSMatt Macy uint64_t blksz = sm->sm_blksz;
89eda14cbcSMatt Macy
90eda14cbcSMatt Macy ASSERT3U(blksz, !=, 0);
91eda14cbcSMatt Macy ASSERT3U(end, <=, space_map_length(sm));
92eda14cbcSMatt Macy ASSERT0(P2PHASE(end, sizeof (uint64_t)));
93eda14cbcSMatt Macy
94eda14cbcSMatt Macy dmu_prefetch(sm->sm_os, space_map_object(sm), 0, 0, end,
95eda14cbcSMatt Macy ZIO_PRIORITY_SYNC_READ);
96eda14cbcSMatt Macy
97eda14cbcSMatt Macy int error = 0;
98eda14cbcSMatt Macy uint64_t txg = 0, sync_pass = 0;
99eda14cbcSMatt Macy for (uint64_t block_base = 0; block_base < end && error == 0;
100eda14cbcSMatt Macy block_base += blksz) {
101eda14cbcSMatt Macy dmu_buf_t *db;
102eda14cbcSMatt Macy error = dmu_buf_hold(sm->sm_os, space_map_object(sm),
103eda14cbcSMatt Macy block_base, FTAG, &db, DMU_READ_PREFETCH);
104eda14cbcSMatt Macy if (error != 0)
105eda14cbcSMatt Macy return (error);
106eda14cbcSMatt Macy
107eda14cbcSMatt Macy uint64_t *block_start = db->db_data;
108eda14cbcSMatt Macy uint64_t block_length = MIN(end - block_base, blksz);
109eda14cbcSMatt Macy uint64_t *block_end = block_start +
110eda14cbcSMatt Macy (block_length / sizeof (uint64_t));
111eda14cbcSMatt Macy
112eda14cbcSMatt Macy VERIFY0(P2PHASE(block_length, sizeof (uint64_t)));
113eda14cbcSMatt Macy VERIFY3U(block_length, !=, 0);
114eda14cbcSMatt Macy ASSERT3U(blksz, ==, db->db_size);
115eda14cbcSMatt Macy
116eda14cbcSMatt Macy for (uint64_t *block_cursor = block_start;
117eda14cbcSMatt Macy block_cursor < block_end && error == 0; block_cursor++) {
118eda14cbcSMatt Macy uint64_t e = *block_cursor;
119eda14cbcSMatt Macy
120eda14cbcSMatt Macy if (sm_entry_is_debug(e)) {
121eda14cbcSMatt Macy /*
122eda14cbcSMatt Macy * Debug entries are only needed to record the
123eda14cbcSMatt Macy * current TXG and sync pass if available.
124eda14cbcSMatt Macy *
125eda14cbcSMatt Macy * Note though that sometimes there can be
126eda14cbcSMatt Macy * debug entries that are used as padding
127eda14cbcSMatt Macy * at the end of space map blocks in-order
128eda14cbcSMatt Macy * to not split a double-word entry in the
129eda14cbcSMatt Macy * middle between two blocks. These entries
130eda14cbcSMatt Macy * have their TXG field set to 0 and we
131eda14cbcSMatt Macy * skip them without recording the TXG.
132eda14cbcSMatt Macy * [see comment in space_map_write_seg()]
133eda14cbcSMatt Macy */
134eda14cbcSMatt Macy uint64_t e_txg = SM_DEBUG_TXG_DECODE(e);
135eda14cbcSMatt Macy if (e_txg != 0) {
136eda14cbcSMatt Macy txg = e_txg;
137eda14cbcSMatt Macy sync_pass = SM_DEBUG_SYNCPASS_DECODE(e);
138eda14cbcSMatt Macy } else {
139eda14cbcSMatt Macy ASSERT0(SM_DEBUG_SYNCPASS_DECODE(e));
140eda14cbcSMatt Macy }
141eda14cbcSMatt Macy continue;
142eda14cbcSMatt Macy }
143eda14cbcSMatt Macy
144eda14cbcSMatt Macy uint64_t raw_offset, raw_run, vdev_id;
145eda14cbcSMatt Macy maptype_t type;
146eda14cbcSMatt Macy if (sm_entry_is_single_word(e)) {
147eda14cbcSMatt Macy type = SM_TYPE_DECODE(e);
148eda14cbcSMatt Macy vdev_id = SM_NO_VDEVID;
149eda14cbcSMatt Macy raw_offset = SM_OFFSET_DECODE(e);
150eda14cbcSMatt Macy raw_run = SM_RUN_DECODE(e);
151eda14cbcSMatt Macy } else {
152eda14cbcSMatt Macy /* it is a two-word entry */
153eda14cbcSMatt Macy ASSERT(sm_entry_is_double_word(e));
154eda14cbcSMatt Macy raw_run = SM2_RUN_DECODE(e);
155eda14cbcSMatt Macy vdev_id = SM2_VDEV_DECODE(e);
156eda14cbcSMatt Macy
157eda14cbcSMatt Macy /* move on to the second word */
158eda14cbcSMatt Macy block_cursor++;
159eda14cbcSMatt Macy e = *block_cursor;
160eda14cbcSMatt Macy VERIFY3P(block_cursor, <=, block_end);
161eda14cbcSMatt Macy
162eda14cbcSMatt Macy type = SM2_TYPE_DECODE(e);
163eda14cbcSMatt Macy raw_offset = SM2_OFFSET_DECODE(e);
164eda14cbcSMatt Macy }
165eda14cbcSMatt Macy
166eda14cbcSMatt Macy uint64_t entry_offset = (raw_offset << sm->sm_shift) +
167eda14cbcSMatt Macy sm->sm_start;
168eda14cbcSMatt Macy uint64_t entry_run = raw_run << sm->sm_shift;
169eda14cbcSMatt Macy
170eda14cbcSMatt Macy VERIFY0(P2PHASE(entry_offset, 1ULL << sm->sm_shift));
171eda14cbcSMatt Macy VERIFY0(P2PHASE(entry_run, 1ULL << sm->sm_shift));
172eda14cbcSMatt Macy ASSERT3U(entry_offset, >=, sm->sm_start);
173eda14cbcSMatt Macy ASSERT3U(entry_offset, <, sm->sm_start + sm->sm_size);
174eda14cbcSMatt Macy ASSERT3U(entry_run, <=, sm->sm_size);
175eda14cbcSMatt Macy ASSERT3U(entry_offset + entry_run, <=,
176eda14cbcSMatt Macy sm->sm_start + sm->sm_size);
177eda14cbcSMatt Macy
178eda14cbcSMatt Macy space_map_entry_t sme = {
179eda14cbcSMatt Macy .sme_type = type,
180eda14cbcSMatt Macy .sme_vdev = vdev_id,
181eda14cbcSMatt Macy .sme_offset = entry_offset,
182eda14cbcSMatt Macy .sme_run = entry_run,
183eda14cbcSMatt Macy .sme_txg = txg,
184eda14cbcSMatt Macy .sme_sync_pass = sync_pass
185eda14cbcSMatt Macy };
186eda14cbcSMatt Macy error = callback(&sme, arg);
187eda14cbcSMatt Macy }
188eda14cbcSMatt Macy dmu_buf_rele(db, FTAG);
189eda14cbcSMatt Macy }
190eda14cbcSMatt Macy return (error);
191eda14cbcSMatt Macy }
192eda14cbcSMatt Macy
193eda14cbcSMatt Macy /*
194eda14cbcSMatt Macy * Reads the entries from the last block of the space map into
195eda14cbcSMatt Macy * buf in reverse order. Populates nwords with number of words
196eda14cbcSMatt Macy * in the last block.
197eda14cbcSMatt Macy *
198eda14cbcSMatt Macy * Refer to block comment within space_map_incremental_destroy()
199eda14cbcSMatt Macy * to understand why this function is needed.
200eda14cbcSMatt Macy */
201eda14cbcSMatt Macy static int
space_map_reversed_last_block_entries(space_map_t * sm,uint64_t * buf,uint64_t bufsz,uint64_t * nwords)202eda14cbcSMatt Macy space_map_reversed_last_block_entries(space_map_t *sm, uint64_t *buf,
203eda14cbcSMatt Macy uint64_t bufsz, uint64_t *nwords)
204eda14cbcSMatt Macy {
205eda14cbcSMatt Macy int error = 0;
206eda14cbcSMatt Macy dmu_buf_t *db;
207eda14cbcSMatt Macy
208eda14cbcSMatt Macy /*
209eda14cbcSMatt Macy * Find the offset of the last word in the space map and use
210eda14cbcSMatt Macy * that to read the last block of the space map with
211eda14cbcSMatt Macy * dmu_buf_hold().
212eda14cbcSMatt Macy */
213eda14cbcSMatt Macy uint64_t last_word_offset =
214eda14cbcSMatt Macy sm->sm_phys->smp_length - sizeof (uint64_t);
215eda14cbcSMatt Macy error = dmu_buf_hold(sm->sm_os, space_map_object(sm), last_word_offset,
216eda14cbcSMatt Macy FTAG, &db, DMU_READ_NO_PREFETCH);
217eda14cbcSMatt Macy if (error != 0)
218eda14cbcSMatt Macy return (error);
219eda14cbcSMatt Macy
220eda14cbcSMatt Macy ASSERT3U(sm->sm_object, ==, db->db_object);
221eda14cbcSMatt Macy ASSERT3U(sm->sm_blksz, ==, db->db_size);
222eda14cbcSMatt Macy ASSERT3U(bufsz, >=, db->db_size);
223eda14cbcSMatt Macy ASSERT(nwords != NULL);
224eda14cbcSMatt Macy
225eda14cbcSMatt Macy uint64_t *words = db->db_data;
226eda14cbcSMatt Macy *nwords =
227eda14cbcSMatt Macy (sm->sm_phys->smp_length - db->db_offset) / sizeof (uint64_t);
228eda14cbcSMatt Macy
229eda14cbcSMatt Macy ASSERT3U(*nwords, <=, bufsz / sizeof (uint64_t));
230eda14cbcSMatt Macy
231eda14cbcSMatt Macy uint64_t n = *nwords;
232eda14cbcSMatt Macy uint64_t j = n - 1;
233eda14cbcSMatt Macy for (uint64_t i = 0; i < n; i++) {
234eda14cbcSMatt Macy uint64_t entry = words[i];
235eda14cbcSMatt Macy if (sm_entry_is_double_word(entry)) {
236eda14cbcSMatt Macy /*
237eda14cbcSMatt Macy * Since we are populating the buffer backwards
238eda14cbcSMatt Macy * we have to be extra careful and add the two
239eda14cbcSMatt Macy * words of the double-word entry in the right
240eda14cbcSMatt Macy * order.
241eda14cbcSMatt Macy */
242eda14cbcSMatt Macy ASSERT3U(j, >, 0);
243eda14cbcSMatt Macy buf[j - 1] = entry;
244eda14cbcSMatt Macy
245eda14cbcSMatt Macy i++;
246eda14cbcSMatt Macy ASSERT3U(i, <, n);
247eda14cbcSMatt Macy entry = words[i];
248eda14cbcSMatt Macy buf[j] = entry;
249eda14cbcSMatt Macy j -= 2;
250eda14cbcSMatt Macy } else {
251eda14cbcSMatt Macy ASSERT(sm_entry_is_debug(entry) ||
252eda14cbcSMatt Macy sm_entry_is_single_word(entry));
253eda14cbcSMatt Macy buf[j] = entry;
254eda14cbcSMatt Macy j--;
255eda14cbcSMatt Macy }
256eda14cbcSMatt Macy }
257eda14cbcSMatt Macy
258eda14cbcSMatt Macy /*
259eda14cbcSMatt Macy * Assert that we wrote backwards all the
260eda14cbcSMatt Macy * way to the beginning of the buffer.
261eda14cbcSMatt Macy */
262eda14cbcSMatt Macy ASSERT3S(j, ==, -1);
263eda14cbcSMatt Macy
264eda14cbcSMatt Macy dmu_buf_rele(db, FTAG);
265eda14cbcSMatt Macy return (error);
266eda14cbcSMatt Macy }
267eda14cbcSMatt Macy
268eda14cbcSMatt Macy /*
269eda14cbcSMatt Macy * Note: This function performs destructive actions - specifically
270eda14cbcSMatt Macy * it deletes entries from the end of the space map. Thus, callers
271eda14cbcSMatt Macy * should ensure that they are holding the appropriate locks for
272eda14cbcSMatt Macy * the space map that they provide.
273eda14cbcSMatt Macy */
274eda14cbcSMatt Macy int
space_map_incremental_destroy(space_map_t * sm,sm_cb_t callback,void * arg,dmu_tx_t * tx)275eda14cbcSMatt Macy space_map_incremental_destroy(space_map_t *sm, sm_cb_t callback, void *arg,
276eda14cbcSMatt Macy dmu_tx_t *tx)
277eda14cbcSMatt Macy {
278eda14cbcSMatt Macy uint64_t bufsz = MAX(sm->sm_blksz, SPA_MINBLOCKSIZE);
279eda14cbcSMatt Macy uint64_t *buf = zio_buf_alloc(bufsz);
280eda14cbcSMatt Macy
281eda14cbcSMatt Macy dmu_buf_will_dirty(sm->sm_dbuf, tx);
282eda14cbcSMatt Macy
283eda14cbcSMatt Macy /*
284eda14cbcSMatt Macy * Ideally we would want to iterate from the beginning of the
285eda14cbcSMatt Macy * space map to the end in incremental steps. The issue with this
286eda14cbcSMatt Macy * approach is that we don't have any field on-disk that points
287eda14cbcSMatt Macy * us where to start between each step. We could try zeroing out
288eda14cbcSMatt Macy * entries that we've destroyed, but this doesn't work either as
289eda14cbcSMatt Macy * an entry that is 0 is a valid one (ALLOC for range [0x0:0x200]).
290eda14cbcSMatt Macy *
291eda14cbcSMatt Macy * As a result, we destroy its entries incrementally starting from
292eda14cbcSMatt Macy * the end after applying the callback to each of them.
293eda14cbcSMatt Macy *
294eda14cbcSMatt Macy * The problem with this approach is that we cannot literally
295eda14cbcSMatt Macy * iterate through the words in the space map backwards as we
296eda14cbcSMatt Macy * can't distinguish two-word space map entries from their second
297eda14cbcSMatt Macy * word. Thus we do the following:
298eda14cbcSMatt Macy *
299eda14cbcSMatt Macy * 1] We get all the entries from the last block of the space map
300eda14cbcSMatt Macy * and put them into a buffer in reverse order. This way the
301eda14cbcSMatt Macy * last entry comes first in the buffer, the second to last is
302eda14cbcSMatt Macy * second, etc.
303eda14cbcSMatt Macy * 2] We iterate through the entries in the buffer and we apply
304eda14cbcSMatt Macy * the callback to each one. As we move from entry to entry we
305eda14cbcSMatt Macy * we decrease the size of the space map, deleting effectively
306eda14cbcSMatt Macy * each entry.
307eda14cbcSMatt Macy * 3] If there are no more entries in the space map or the callback
308eda14cbcSMatt Macy * returns a value other than 0, we stop iterating over the
309eda14cbcSMatt Macy * space map. If there are entries remaining and the callback
310eda14cbcSMatt Macy * returned 0, we go back to step [1].
311eda14cbcSMatt Macy */
312eda14cbcSMatt Macy int error = 0;
313eda14cbcSMatt Macy while (space_map_length(sm) > 0 && error == 0) {
314eda14cbcSMatt Macy uint64_t nwords = 0;
315eda14cbcSMatt Macy error = space_map_reversed_last_block_entries(sm, buf, bufsz,
316eda14cbcSMatt Macy &nwords);
317eda14cbcSMatt Macy if (error != 0)
318eda14cbcSMatt Macy break;
319eda14cbcSMatt Macy
320eda14cbcSMatt Macy ASSERT3U(nwords, <=, bufsz / sizeof (uint64_t));
321eda14cbcSMatt Macy
322eda14cbcSMatt Macy for (uint64_t i = 0; i < nwords; i++) {
323eda14cbcSMatt Macy uint64_t e = buf[i];
324eda14cbcSMatt Macy
325eda14cbcSMatt Macy if (sm_entry_is_debug(e)) {
326eda14cbcSMatt Macy sm->sm_phys->smp_length -= sizeof (uint64_t);
327eda14cbcSMatt Macy continue;
328eda14cbcSMatt Macy }
329eda14cbcSMatt Macy
330eda14cbcSMatt Macy int words = 1;
331eda14cbcSMatt Macy uint64_t raw_offset, raw_run, vdev_id;
332eda14cbcSMatt Macy maptype_t type;
333eda14cbcSMatt Macy if (sm_entry_is_single_word(e)) {
334eda14cbcSMatt Macy type = SM_TYPE_DECODE(e);
335eda14cbcSMatt Macy vdev_id = SM_NO_VDEVID;
336eda14cbcSMatt Macy raw_offset = SM_OFFSET_DECODE(e);
337eda14cbcSMatt Macy raw_run = SM_RUN_DECODE(e);
338eda14cbcSMatt Macy } else {
339eda14cbcSMatt Macy ASSERT(sm_entry_is_double_word(e));
340eda14cbcSMatt Macy words = 2;
341eda14cbcSMatt Macy
342eda14cbcSMatt Macy raw_run = SM2_RUN_DECODE(e);
343eda14cbcSMatt Macy vdev_id = SM2_VDEV_DECODE(e);
344eda14cbcSMatt Macy
345eda14cbcSMatt Macy /* move to the second word */
346eda14cbcSMatt Macy i++;
347eda14cbcSMatt Macy e = buf[i];
348eda14cbcSMatt Macy
349eda14cbcSMatt Macy ASSERT3P(i, <=, nwords);
350eda14cbcSMatt Macy
351eda14cbcSMatt Macy type = SM2_TYPE_DECODE(e);
352eda14cbcSMatt Macy raw_offset = SM2_OFFSET_DECODE(e);
353eda14cbcSMatt Macy }
354eda14cbcSMatt Macy
355eda14cbcSMatt Macy uint64_t entry_offset =
356eda14cbcSMatt Macy (raw_offset << sm->sm_shift) + sm->sm_start;
357eda14cbcSMatt Macy uint64_t entry_run = raw_run << sm->sm_shift;
358eda14cbcSMatt Macy
359eda14cbcSMatt Macy VERIFY0(P2PHASE(entry_offset, 1ULL << sm->sm_shift));
360eda14cbcSMatt Macy VERIFY0(P2PHASE(entry_run, 1ULL << sm->sm_shift));
361eda14cbcSMatt Macy VERIFY3U(entry_offset, >=, sm->sm_start);
362eda14cbcSMatt Macy VERIFY3U(entry_offset, <, sm->sm_start + sm->sm_size);
363eda14cbcSMatt Macy VERIFY3U(entry_run, <=, sm->sm_size);
364eda14cbcSMatt Macy VERIFY3U(entry_offset + entry_run, <=,
365eda14cbcSMatt Macy sm->sm_start + sm->sm_size);
366eda14cbcSMatt Macy
367eda14cbcSMatt Macy space_map_entry_t sme = {
368eda14cbcSMatt Macy .sme_type = type,
369eda14cbcSMatt Macy .sme_vdev = vdev_id,
370eda14cbcSMatt Macy .sme_offset = entry_offset,
371eda14cbcSMatt Macy .sme_run = entry_run
372eda14cbcSMatt Macy };
373eda14cbcSMatt Macy error = callback(&sme, arg);
374eda14cbcSMatt Macy if (error != 0)
375eda14cbcSMatt Macy break;
376eda14cbcSMatt Macy
377eda14cbcSMatt Macy if (type == SM_ALLOC)
378eda14cbcSMatt Macy sm->sm_phys->smp_alloc -= entry_run;
379eda14cbcSMatt Macy else
380eda14cbcSMatt Macy sm->sm_phys->smp_alloc += entry_run;
381eda14cbcSMatt Macy sm->sm_phys->smp_length -= words * sizeof (uint64_t);
382eda14cbcSMatt Macy }
383eda14cbcSMatt Macy }
384eda14cbcSMatt Macy
385eda14cbcSMatt Macy if (space_map_length(sm) == 0) {
386eda14cbcSMatt Macy ASSERT0(error);
387eda14cbcSMatt Macy ASSERT0(space_map_allocated(sm));
388eda14cbcSMatt Macy }
389eda14cbcSMatt Macy
390eda14cbcSMatt Macy zio_buf_free(buf, bufsz);
391eda14cbcSMatt Macy return (error);
392eda14cbcSMatt Macy }
393eda14cbcSMatt Macy
394eda14cbcSMatt Macy typedef struct space_map_load_arg {
395eda14cbcSMatt Macy space_map_t *smla_sm;
396eda14cbcSMatt Macy range_tree_t *smla_rt;
397eda14cbcSMatt Macy maptype_t smla_type;
398eda14cbcSMatt Macy } space_map_load_arg_t;
399eda14cbcSMatt Macy
400eda14cbcSMatt Macy static int
space_map_load_callback(space_map_entry_t * sme,void * arg)401eda14cbcSMatt Macy space_map_load_callback(space_map_entry_t *sme, void *arg)
402eda14cbcSMatt Macy {
403eda14cbcSMatt Macy space_map_load_arg_t *smla = arg;
404eda14cbcSMatt Macy if (sme->sme_type == smla->smla_type) {
405eda14cbcSMatt Macy VERIFY3U(range_tree_space(smla->smla_rt) + sme->sme_run, <=,
406eda14cbcSMatt Macy smla->smla_sm->sm_size);
407eda14cbcSMatt Macy range_tree_add(smla->smla_rt, sme->sme_offset, sme->sme_run);
408eda14cbcSMatt Macy } else {
409eda14cbcSMatt Macy range_tree_remove(smla->smla_rt, sme->sme_offset, sme->sme_run);
410eda14cbcSMatt Macy }
411eda14cbcSMatt Macy
412eda14cbcSMatt Macy return (0);
413eda14cbcSMatt Macy }
414eda14cbcSMatt Macy
415eda14cbcSMatt Macy /*
416eda14cbcSMatt Macy * Load the spacemap into the rangetree, like space_map_load. But only
417eda14cbcSMatt Macy * read the first 'length' bytes of the spacemap.
418eda14cbcSMatt Macy */
419eda14cbcSMatt Macy int
space_map_load_length(space_map_t * sm,range_tree_t * rt,maptype_t maptype,uint64_t length)420eda14cbcSMatt Macy space_map_load_length(space_map_t *sm, range_tree_t *rt, maptype_t maptype,
421eda14cbcSMatt Macy uint64_t length)
422eda14cbcSMatt Macy {
423eda14cbcSMatt Macy space_map_load_arg_t smla;
424eda14cbcSMatt Macy
425eda14cbcSMatt Macy VERIFY0(range_tree_space(rt));
426eda14cbcSMatt Macy
427eda14cbcSMatt Macy if (maptype == SM_FREE)
428eda14cbcSMatt Macy range_tree_add(rt, sm->sm_start, sm->sm_size);
429eda14cbcSMatt Macy
430eda14cbcSMatt Macy smla.smla_rt = rt;
431eda14cbcSMatt Macy smla.smla_sm = sm;
432eda14cbcSMatt Macy smla.smla_type = maptype;
433eda14cbcSMatt Macy int err = space_map_iterate(sm, length,
434eda14cbcSMatt Macy space_map_load_callback, &smla);
435eda14cbcSMatt Macy
436eda14cbcSMatt Macy if (err != 0)
437eda14cbcSMatt Macy range_tree_vacate(rt, NULL, NULL);
438eda14cbcSMatt Macy
439eda14cbcSMatt Macy return (err);
440eda14cbcSMatt Macy }
441eda14cbcSMatt Macy
442eda14cbcSMatt Macy /*
443eda14cbcSMatt Macy * Load the space map disk into the specified range tree. Segments of maptype
444eda14cbcSMatt Macy * are added to the range tree, other segment types are removed.
445eda14cbcSMatt Macy */
446eda14cbcSMatt Macy int
space_map_load(space_map_t * sm,range_tree_t * rt,maptype_t maptype)447eda14cbcSMatt Macy space_map_load(space_map_t *sm, range_tree_t *rt, maptype_t maptype)
448eda14cbcSMatt Macy {
449eda14cbcSMatt Macy return (space_map_load_length(sm, rt, maptype, space_map_length(sm)));
450eda14cbcSMatt Macy }
451eda14cbcSMatt Macy
452eda14cbcSMatt Macy void
space_map_histogram_clear(space_map_t * sm)453eda14cbcSMatt Macy space_map_histogram_clear(space_map_t *sm)
454eda14cbcSMatt Macy {
455eda14cbcSMatt Macy if (sm->sm_dbuf->db_size != sizeof (space_map_phys_t))
456eda14cbcSMatt Macy return;
457eda14cbcSMatt Macy
458da5137abSMartin Matuska memset(sm->sm_phys->smp_histogram, 0,
459da5137abSMartin Matuska sizeof (sm->sm_phys->smp_histogram));
460eda14cbcSMatt Macy }
461eda14cbcSMatt Macy
462eda14cbcSMatt Macy boolean_t
space_map_histogram_verify(space_map_t * sm,range_tree_t * rt)463eda14cbcSMatt Macy space_map_histogram_verify(space_map_t *sm, range_tree_t *rt)
464eda14cbcSMatt Macy {
465eda14cbcSMatt Macy /*
466eda14cbcSMatt Macy * Verify that the in-core range tree does not have any
467eda14cbcSMatt Macy * ranges smaller than our sm_shift size.
468eda14cbcSMatt Macy */
469eda14cbcSMatt Macy for (int i = 0; i < sm->sm_shift; i++) {
470eda14cbcSMatt Macy if (rt->rt_histogram[i] != 0)
471eda14cbcSMatt Macy return (B_FALSE);
472eda14cbcSMatt Macy }
473eda14cbcSMatt Macy return (B_TRUE);
474eda14cbcSMatt Macy }
475eda14cbcSMatt Macy
476eda14cbcSMatt Macy void
space_map_histogram_add(space_map_t * sm,range_tree_t * rt,dmu_tx_t * tx)477eda14cbcSMatt Macy space_map_histogram_add(space_map_t *sm, range_tree_t *rt, dmu_tx_t *tx)
478eda14cbcSMatt Macy {
479eda14cbcSMatt Macy int idx = 0;
480eda14cbcSMatt Macy
481eda14cbcSMatt Macy ASSERT(dmu_tx_is_syncing(tx));
482eda14cbcSMatt Macy VERIFY3U(space_map_object(sm), !=, 0);
483eda14cbcSMatt Macy
484eda14cbcSMatt Macy if (sm->sm_dbuf->db_size != sizeof (space_map_phys_t))
485eda14cbcSMatt Macy return;
486eda14cbcSMatt Macy
487eda14cbcSMatt Macy dmu_buf_will_dirty(sm->sm_dbuf, tx);
488eda14cbcSMatt Macy
489eda14cbcSMatt Macy ASSERT(space_map_histogram_verify(sm, rt));
490eda14cbcSMatt Macy /*
491eda14cbcSMatt Macy * Transfer the content of the range tree histogram to the space
492eda14cbcSMatt Macy * map histogram. The space map histogram contains 32 buckets ranging
493eda14cbcSMatt Macy * between 2^sm_shift to 2^(32+sm_shift-1). The range tree,
494eda14cbcSMatt Macy * however, can represent ranges from 2^0 to 2^63. Since the space
495eda14cbcSMatt Macy * map only cares about allocatable blocks (minimum of sm_shift) we
496eda14cbcSMatt Macy * can safely ignore all ranges in the range tree smaller than sm_shift.
497eda14cbcSMatt Macy */
498eda14cbcSMatt Macy for (int i = sm->sm_shift; i < RANGE_TREE_HISTOGRAM_SIZE; i++) {
499eda14cbcSMatt Macy
500eda14cbcSMatt Macy /*
501eda14cbcSMatt Macy * Since the largest histogram bucket in the space map is
502eda14cbcSMatt Macy * 2^(32+sm_shift-1), we need to normalize the values in
503eda14cbcSMatt Macy * the range tree for any bucket larger than that size. For
504eda14cbcSMatt Macy * example given an sm_shift of 9, ranges larger than 2^40
505eda14cbcSMatt Macy * would get normalized as if they were 1TB ranges. Assume
506eda14cbcSMatt Macy * the range tree had a count of 5 in the 2^44 (16TB) bucket,
507eda14cbcSMatt Macy * the calculation below would normalize this to 5 * 2^4 (16).
508eda14cbcSMatt Macy */
509eda14cbcSMatt Macy ASSERT3U(i, >=, idx + sm->sm_shift);
510eda14cbcSMatt Macy sm->sm_phys->smp_histogram[idx] +=
511eda14cbcSMatt Macy rt->rt_histogram[i] << (i - idx - sm->sm_shift);
512eda14cbcSMatt Macy
513eda14cbcSMatt Macy /*
514eda14cbcSMatt Macy * Increment the space map's index as long as we haven't
515eda14cbcSMatt Macy * reached the maximum bucket size. Accumulate all ranges
516eda14cbcSMatt Macy * larger than the max bucket size into the last bucket.
517eda14cbcSMatt Macy */
518eda14cbcSMatt Macy if (idx < SPACE_MAP_HISTOGRAM_SIZE - 1) {
519eda14cbcSMatt Macy ASSERT3U(idx + sm->sm_shift, ==, i);
520eda14cbcSMatt Macy idx++;
521eda14cbcSMatt Macy ASSERT3U(idx, <, SPACE_MAP_HISTOGRAM_SIZE);
522eda14cbcSMatt Macy }
523eda14cbcSMatt Macy }
524eda14cbcSMatt Macy }
525eda14cbcSMatt Macy
526eda14cbcSMatt Macy static void
space_map_write_intro_debug(space_map_t * sm,maptype_t maptype,dmu_tx_t * tx)527eda14cbcSMatt Macy space_map_write_intro_debug(space_map_t *sm, maptype_t maptype, dmu_tx_t *tx)
528eda14cbcSMatt Macy {
529eda14cbcSMatt Macy dmu_buf_will_dirty(sm->sm_dbuf, tx);
530eda14cbcSMatt Macy
531eda14cbcSMatt Macy uint64_t dentry = SM_PREFIX_ENCODE(SM_DEBUG_PREFIX) |
532eda14cbcSMatt Macy SM_DEBUG_ACTION_ENCODE(maptype) |
533eda14cbcSMatt Macy SM_DEBUG_SYNCPASS_ENCODE(spa_sync_pass(tx->tx_pool->dp_spa)) |
534eda14cbcSMatt Macy SM_DEBUG_TXG_ENCODE(dmu_tx_get_txg(tx));
535eda14cbcSMatt Macy
536eda14cbcSMatt Macy dmu_write(sm->sm_os, space_map_object(sm), sm->sm_phys->smp_length,
537eda14cbcSMatt Macy sizeof (dentry), &dentry, tx);
538eda14cbcSMatt Macy
539eda14cbcSMatt Macy sm->sm_phys->smp_length += sizeof (dentry);
540eda14cbcSMatt Macy }
541eda14cbcSMatt Macy
542eda14cbcSMatt Macy /*
543eda14cbcSMatt Macy * Writes one or more entries given a segment.
544eda14cbcSMatt Macy *
545eda14cbcSMatt Macy * Note: The function may release the dbuf from the pointer initially
546eda14cbcSMatt Macy * passed to it, and return a different dbuf. Also, the space map's
547eda14cbcSMatt Macy * dbuf must be dirty for the changes in sm_phys to take effect.
548eda14cbcSMatt Macy */
549eda14cbcSMatt Macy static void
space_map_write_seg(space_map_t * sm,uint64_t rstart,uint64_t rend,maptype_t maptype,uint64_t vdev_id,uint8_t words,dmu_buf_t ** dbp,const void * tag,dmu_tx_t * tx)550eda14cbcSMatt Macy space_map_write_seg(space_map_t *sm, uint64_t rstart, uint64_t rend,
551eda14cbcSMatt Macy maptype_t maptype, uint64_t vdev_id, uint8_t words, dmu_buf_t **dbp,
552a0b956f5SMartin Matuska const void *tag, dmu_tx_t *tx)
553eda14cbcSMatt Macy {
554eda14cbcSMatt Macy ASSERT3U(words, !=, 0);
555eda14cbcSMatt Macy ASSERT3U(words, <=, 2);
556eda14cbcSMatt Macy
557eda14cbcSMatt Macy /* ensure the vdev_id can be represented by the space map */
558eda14cbcSMatt Macy ASSERT3U(vdev_id, <=, SM_NO_VDEVID);
559eda14cbcSMatt Macy
560eda14cbcSMatt Macy /*
561eda14cbcSMatt Macy * if this is a single word entry, ensure that no vdev was
562eda14cbcSMatt Macy * specified.
563eda14cbcSMatt Macy */
564eda14cbcSMatt Macy IMPLY(words == 1, vdev_id == SM_NO_VDEVID);
565eda14cbcSMatt Macy
566eda14cbcSMatt Macy dmu_buf_t *db = *dbp;
567eda14cbcSMatt Macy ASSERT3U(db->db_size, ==, sm->sm_blksz);
568eda14cbcSMatt Macy
569eda14cbcSMatt Macy uint64_t *block_base = db->db_data;
570eda14cbcSMatt Macy uint64_t *block_end = block_base + (sm->sm_blksz / sizeof (uint64_t));
571eda14cbcSMatt Macy uint64_t *block_cursor = block_base +
572eda14cbcSMatt Macy (sm->sm_phys->smp_length - db->db_offset) / sizeof (uint64_t);
573eda14cbcSMatt Macy
574eda14cbcSMatt Macy ASSERT3P(block_cursor, <=, block_end);
575eda14cbcSMatt Macy
576eda14cbcSMatt Macy uint64_t size = (rend - rstart) >> sm->sm_shift;
577eda14cbcSMatt Macy uint64_t start = (rstart - sm->sm_start) >> sm->sm_shift;
578eda14cbcSMatt Macy uint64_t run_max = (words == 2) ? SM2_RUN_MAX : SM_RUN_MAX;
579eda14cbcSMatt Macy
580eda14cbcSMatt Macy ASSERT3U(rstart, >=, sm->sm_start);
581eda14cbcSMatt Macy ASSERT3U(rstart, <, sm->sm_start + sm->sm_size);
582eda14cbcSMatt Macy ASSERT3U(rend - rstart, <=, sm->sm_size);
583eda14cbcSMatt Macy ASSERT3U(rend, <=, sm->sm_start + sm->sm_size);
584eda14cbcSMatt Macy
585eda14cbcSMatt Macy while (size != 0) {
586eda14cbcSMatt Macy ASSERT3P(block_cursor, <=, block_end);
587eda14cbcSMatt Macy
588eda14cbcSMatt Macy /*
589eda14cbcSMatt Macy * If we are at the end of this block, flush it and start
590eda14cbcSMatt Macy * writing again from the beginning.
591eda14cbcSMatt Macy */
592eda14cbcSMatt Macy if (block_cursor == block_end) {
593eda14cbcSMatt Macy dmu_buf_rele(db, tag);
594eda14cbcSMatt Macy
595eda14cbcSMatt Macy uint64_t next_word_offset = sm->sm_phys->smp_length;
596eda14cbcSMatt Macy VERIFY0(dmu_buf_hold(sm->sm_os,
597eda14cbcSMatt Macy space_map_object(sm), next_word_offset,
598eda14cbcSMatt Macy tag, &db, DMU_READ_PREFETCH));
599eda14cbcSMatt Macy dmu_buf_will_dirty(db, tx);
600eda14cbcSMatt Macy
601eda14cbcSMatt Macy /* update caller's dbuf */
602eda14cbcSMatt Macy *dbp = db;
603eda14cbcSMatt Macy
604eda14cbcSMatt Macy ASSERT3U(db->db_size, ==, sm->sm_blksz);
605eda14cbcSMatt Macy
606eda14cbcSMatt Macy block_base = db->db_data;
607eda14cbcSMatt Macy block_cursor = block_base;
608eda14cbcSMatt Macy block_end = block_base +
609eda14cbcSMatt Macy (db->db_size / sizeof (uint64_t));
610eda14cbcSMatt Macy }
611eda14cbcSMatt Macy
612eda14cbcSMatt Macy /*
613eda14cbcSMatt Macy * If we are writing a two-word entry and we only have one
614eda14cbcSMatt Macy * word left on this block, just pad it with an empty debug
615eda14cbcSMatt Macy * entry and write the two-word entry in the next block.
616eda14cbcSMatt Macy */
617eda14cbcSMatt Macy uint64_t *next_entry = block_cursor + 1;
618eda14cbcSMatt Macy if (next_entry == block_end && words > 1) {
619eda14cbcSMatt Macy ASSERT3U(words, ==, 2);
620eda14cbcSMatt Macy *block_cursor = SM_PREFIX_ENCODE(SM_DEBUG_PREFIX) |
621eda14cbcSMatt Macy SM_DEBUG_ACTION_ENCODE(0) |
622eda14cbcSMatt Macy SM_DEBUG_SYNCPASS_ENCODE(0) |
623eda14cbcSMatt Macy SM_DEBUG_TXG_ENCODE(0);
624eda14cbcSMatt Macy block_cursor++;
625eda14cbcSMatt Macy sm->sm_phys->smp_length += sizeof (uint64_t);
626eda14cbcSMatt Macy ASSERT3P(block_cursor, ==, block_end);
627eda14cbcSMatt Macy continue;
628eda14cbcSMatt Macy }
629eda14cbcSMatt Macy
630eda14cbcSMatt Macy uint64_t run_len = MIN(size, run_max);
631eda14cbcSMatt Macy switch (words) {
632eda14cbcSMatt Macy case 1:
633eda14cbcSMatt Macy *block_cursor = SM_OFFSET_ENCODE(start) |
634eda14cbcSMatt Macy SM_TYPE_ENCODE(maptype) |
635eda14cbcSMatt Macy SM_RUN_ENCODE(run_len);
636eda14cbcSMatt Macy block_cursor++;
637eda14cbcSMatt Macy break;
638eda14cbcSMatt Macy case 2:
639eda14cbcSMatt Macy /* write the first word of the entry */
640eda14cbcSMatt Macy *block_cursor = SM_PREFIX_ENCODE(SM2_PREFIX) |
641eda14cbcSMatt Macy SM2_RUN_ENCODE(run_len) |
642eda14cbcSMatt Macy SM2_VDEV_ENCODE(vdev_id);
643eda14cbcSMatt Macy block_cursor++;
644eda14cbcSMatt Macy
645eda14cbcSMatt Macy /* move on to the second word of the entry */
646eda14cbcSMatt Macy ASSERT3P(block_cursor, <, block_end);
647eda14cbcSMatt Macy *block_cursor = SM2_TYPE_ENCODE(maptype) |
648eda14cbcSMatt Macy SM2_OFFSET_ENCODE(start);
649eda14cbcSMatt Macy block_cursor++;
650eda14cbcSMatt Macy break;
651eda14cbcSMatt Macy default:
652eda14cbcSMatt Macy panic("%d-word space map entries are not supported",
653eda14cbcSMatt Macy words);
654eda14cbcSMatt Macy break;
655eda14cbcSMatt Macy }
656eda14cbcSMatt Macy sm->sm_phys->smp_length += words * sizeof (uint64_t);
657eda14cbcSMatt Macy
658eda14cbcSMatt Macy start += run_len;
659eda14cbcSMatt Macy size -= run_len;
660eda14cbcSMatt Macy }
661eda14cbcSMatt Macy ASSERT0(size);
662eda14cbcSMatt Macy
663eda14cbcSMatt Macy }
664eda14cbcSMatt Macy
665eda14cbcSMatt Macy /*
666eda14cbcSMatt Macy * Note: The space map's dbuf must be dirty for the changes in sm_phys to
667eda14cbcSMatt Macy * take effect.
668eda14cbcSMatt Macy */
669eda14cbcSMatt Macy static void
space_map_write_impl(space_map_t * sm,range_tree_t * rt,maptype_t maptype,uint64_t vdev_id,dmu_tx_t * tx)670eda14cbcSMatt Macy space_map_write_impl(space_map_t *sm, range_tree_t *rt, maptype_t maptype,
671eda14cbcSMatt Macy uint64_t vdev_id, dmu_tx_t *tx)
672eda14cbcSMatt Macy {
673eda14cbcSMatt Macy spa_t *spa = tx->tx_pool->dp_spa;
674eda14cbcSMatt Macy dmu_buf_t *db;
675eda14cbcSMatt Macy
676eda14cbcSMatt Macy space_map_write_intro_debug(sm, maptype, tx);
677eda14cbcSMatt Macy
678eda14cbcSMatt Macy #ifdef ZFS_DEBUG
679eda14cbcSMatt Macy /*
680eda14cbcSMatt Macy * We do this right after we write the intro debug entry
681eda14cbcSMatt Macy * because the estimate does not take it into account.
682eda14cbcSMatt Macy */
683eda14cbcSMatt Macy uint64_t initial_objsize = sm->sm_phys->smp_length;
684eda14cbcSMatt Macy uint64_t estimated_growth =
685eda14cbcSMatt Macy space_map_estimate_optimal_size(sm, rt, SM_NO_VDEVID);
686eda14cbcSMatt Macy uint64_t estimated_final_objsize = initial_objsize + estimated_growth;
687eda14cbcSMatt Macy #endif
688eda14cbcSMatt Macy
689eda14cbcSMatt Macy /*
690eda14cbcSMatt Macy * Find the offset right after the last word in the space map
691eda14cbcSMatt Macy * and use that to get a hold of the last block, so we can
692eda14cbcSMatt Macy * start appending to it.
693eda14cbcSMatt Macy */
694eda14cbcSMatt Macy uint64_t next_word_offset = sm->sm_phys->smp_length;
695eda14cbcSMatt Macy VERIFY0(dmu_buf_hold(sm->sm_os, space_map_object(sm),
696eda14cbcSMatt Macy next_word_offset, FTAG, &db, DMU_READ_PREFETCH));
697eda14cbcSMatt Macy ASSERT3U(db->db_size, ==, sm->sm_blksz);
698eda14cbcSMatt Macy
699eda14cbcSMatt Macy dmu_buf_will_dirty(db, tx);
700eda14cbcSMatt Macy
701eda14cbcSMatt Macy zfs_btree_t *t = &rt->rt_root;
702eda14cbcSMatt Macy zfs_btree_index_t where;
703eda14cbcSMatt Macy for (range_seg_t *rs = zfs_btree_first(t, &where); rs != NULL;
704eda14cbcSMatt Macy rs = zfs_btree_next(t, &where, &where)) {
705eda14cbcSMatt Macy uint64_t offset = (rs_get_start(rs, rt) - sm->sm_start) >>
706eda14cbcSMatt Macy sm->sm_shift;
707eda14cbcSMatt Macy uint64_t length = (rs_get_end(rs, rt) - rs_get_start(rs, rt)) >>
708eda14cbcSMatt Macy sm->sm_shift;
709eda14cbcSMatt Macy uint8_t words = 1;
710eda14cbcSMatt Macy
711eda14cbcSMatt Macy /*
712eda14cbcSMatt Macy * We only write two-word entries when both of the following
713eda14cbcSMatt Macy * are true:
714eda14cbcSMatt Macy *
715eda14cbcSMatt Macy * [1] The feature is enabled.
716eda14cbcSMatt Macy * [2] The offset or run is too big for a single-word entry,
717eda14cbcSMatt Macy * or the vdev_id is set (meaning not equal to
718eda14cbcSMatt Macy * SM_NO_VDEVID).
719eda14cbcSMatt Macy *
720eda14cbcSMatt Macy * Note that for purposes of testing we've added the case that
721eda14cbcSMatt Macy * we write two-word entries occasionally when the feature is
722eda14cbcSMatt Macy * enabled and zfs_force_some_double_word_sm_entries has been
723eda14cbcSMatt Macy * set.
724eda14cbcSMatt Macy */
725eda14cbcSMatt Macy if (spa_feature_is_active(spa, SPA_FEATURE_SPACEMAP_V2) &&
726eda14cbcSMatt Macy (offset >= (1ULL << SM_OFFSET_BITS) ||
727eda14cbcSMatt Macy length > SM_RUN_MAX ||
728eda14cbcSMatt Macy vdev_id != SM_NO_VDEVID ||
729eda14cbcSMatt Macy (zfs_force_some_double_word_sm_entries &&
73033b8c039SMartin Matuska random_in_range(100) == 0)))
731eda14cbcSMatt Macy words = 2;
732eda14cbcSMatt Macy
733eda14cbcSMatt Macy space_map_write_seg(sm, rs_get_start(rs, rt), rs_get_end(rs,
734eda14cbcSMatt Macy rt), maptype, vdev_id, words, &db, FTAG, tx);
735eda14cbcSMatt Macy }
736eda14cbcSMatt Macy
737eda14cbcSMatt Macy dmu_buf_rele(db, FTAG);
738eda14cbcSMatt Macy
739eda14cbcSMatt Macy #ifdef ZFS_DEBUG
740eda14cbcSMatt Macy /*
741eda14cbcSMatt Macy * We expect our estimation to be based on the worst case
742eda14cbcSMatt Macy * scenario [see comment in space_map_estimate_optimal_size()].
743eda14cbcSMatt Macy * Therefore we expect the actual objsize to be equal or less
744eda14cbcSMatt Macy * than whatever we estimated it to be.
745eda14cbcSMatt Macy */
746eda14cbcSMatt Macy ASSERT3U(estimated_final_objsize, >=, sm->sm_phys->smp_length);
747eda14cbcSMatt Macy #endif
748eda14cbcSMatt Macy }
749eda14cbcSMatt Macy
750eda14cbcSMatt Macy /*
751eda14cbcSMatt Macy * Note: This function manipulates the state of the given space map but
752eda14cbcSMatt Macy * does not hold any locks implicitly. Thus the caller is responsible
753eda14cbcSMatt Macy * for synchronizing writes to the space map.
754eda14cbcSMatt Macy */
755eda14cbcSMatt Macy void
space_map_write(space_map_t * sm,range_tree_t * rt,maptype_t maptype,uint64_t vdev_id,dmu_tx_t * tx)756eda14cbcSMatt Macy space_map_write(space_map_t *sm, range_tree_t *rt, maptype_t maptype,
757eda14cbcSMatt Macy uint64_t vdev_id, dmu_tx_t *tx)
758eda14cbcSMatt Macy {
759eda14cbcSMatt Macy ASSERT(dsl_pool_sync_context(dmu_objset_pool(sm->sm_os)));
760eda14cbcSMatt Macy VERIFY3U(space_map_object(sm), !=, 0);
761eda14cbcSMatt Macy
762eda14cbcSMatt Macy dmu_buf_will_dirty(sm->sm_dbuf, tx);
763eda14cbcSMatt Macy
764eda14cbcSMatt Macy /*
765eda14cbcSMatt Macy * This field is no longer necessary since the in-core space map
766eda14cbcSMatt Macy * now contains the object number but is maintained for backwards
767eda14cbcSMatt Macy * compatibility.
768eda14cbcSMatt Macy */
769eda14cbcSMatt Macy sm->sm_phys->smp_object = sm->sm_object;
770eda14cbcSMatt Macy
771eda14cbcSMatt Macy if (range_tree_is_empty(rt)) {
772eda14cbcSMatt Macy VERIFY3U(sm->sm_object, ==, sm->sm_phys->smp_object);
773eda14cbcSMatt Macy return;
774eda14cbcSMatt Macy }
775eda14cbcSMatt Macy
776eda14cbcSMatt Macy if (maptype == SM_ALLOC)
777eda14cbcSMatt Macy sm->sm_phys->smp_alloc += range_tree_space(rt);
778eda14cbcSMatt Macy else
779eda14cbcSMatt Macy sm->sm_phys->smp_alloc -= range_tree_space(rt);
780eda14cbcSMatt Macy
781eda14cbcSMatt Macy uint64_t nodes = zfs_btree_numnodes(&rt->rt_root);
782eda14cbcSMatt Macy uint64_t rt_space = range_tree_space(rt);
783eda14cbcSMatt Macy
784eda14cbcSMatt Macy space_map_write_impl(sm, rt, maptype, vdev_id, tx);
785eda14cbcSMatt Macy
786eda14cbcSMatt Macy /*
787eda14cbcSMatt Macy * Ensure that the space_map's accounting wasn't changed
788eda14cbcSMatt Macy * while we were in the middle of writing it out.
789eda14cbcSMatt Macy */
790eda14cbcSMatt Macy VERIFY3U(nodes, ==, zfs_btree_numnodes(&rt->rt_root));
791eda14cbcSMatt Macy VERIFY3U(range_tree_space(rt), ==, rt_space);
792eda14cbcSMatt Macy }
793eda14cbcSMatt Macy
794eda14cbcSMatt Macy static int
space_map_open_impl(space_map_t * sm)795eda14cbcSMatt Macy space_map_open_impl(space_map_t *sm)
796eda14cbcSMatt Macy {
797eda14cbcSMatt Macy int error;
798eda14cbcSMatt Macy u_longlong_t blocks;
799eda14cbcSMatt Macy
800eda14cbcSMatt Macy error = dmu_bonus_hold(sm->sm_os, sm->sm_object, sm, &sm->sm_dbuf);
801eda14cbcSMatt Macy if (error)
802eda14cbcSMatt Macy return (error);
803eda14cbcSMatt Macy
804eda14cbcSMatt Macy dmu_object_size_from_db(sm->sm_dbuf, &sm->sm_blksz, &blocks);
805eda14cbcSMatt Macy sm->sm_phys = sm->sm_dbuf->db_data;
806eda14cbcSMatt Macy return (0);
807eda14cbcSMatt Macy }
808eda14cbcSMatt Macy
809eda14cbcSMatt Macy int
space_map_open(space_map_t ** smp,objset_t * os,uint64_t object,uint64_t start,uint64_t size,uint8_t shift)810eda14cbcSMatt Macy space_map_open(space_map_t **smp, objset_t *os, uint64_t object,
811eda14cbcSMatt Macy uint64_t start, uint64_t size, uint8_t shift)
812eda14cbcSMatt Macy {
813eda14cbcSMatt Macy space_map_t *sm;
814eda14cbcSMatt Macy int error;
815eda14cbcSMatt Macy
816eda14cbcSMatt Macy ASSERT(*smp == NULL);
817eda14cbcSMatt Macy ASSERT(os != NULL);
818eda14cbcSMatt Macy ASSERT(object != 0);
819eda14cbcSMatt Macy
820eda14cbcSMatt Macy sm = kmem_alloc(sizeof (space_map_t), KM_SLEEP);
821eda14cbcSMatt Macy
822eda14cbcSMatt Macy sm->sm_start = start;
823eda14cbcSMatt Macy sm->sm_size = size;
824eda14cbcSMatt Macy sm->sm_shift = shift;
825eda14cbcSMatt Macy sm->sm_os = os;
826eda14cbcSMatt Macy sm->sm_object = object;
827eda14cbcSMatt Macy sm->sm_blksz = 0;
828eda14cbcSMatt Macy sm->sm_dbuf = NULL;
829eda14cbcSMatt Macy sm->sm_phys = NULL;
830eda14cbcSMatt Macy
831eda14cbcSMatt Macy error = space_map_open_impl(sm);
832eda14cbcSMatt Macy if (error != 0) {
833eda14cbcSMatt Macy space_map_close(sm);
834eda14cbcSMatt Macy return (error);
835eda14cbcSMatt Macy }
836eda14cbcSMatt Macy *smp = sm;
837eda14cbcSMatt Macy
838eda14cbcSMatt Macy return (0);
839eda14cbcSMatt Macy }
840eda14cbcSMatt Macy
841eda14cbcSMatt Macy void
space_map_close(space_map_t * sm)842eda14cbcSMatt Macy space_map_close(space_map_t *sm)
843eda14cbcSMatt Macy {
844eda14cbcSMatt Macy if (sm == NULL)
845eda14cbcSMatt Macy return;
846eda14cbcSMatt Macy
847eda14cbcSMatt Macy if (sm->sm_dbuf != NULL)
848eda14cbcSMatt Macy dmu_buf_rele(sm->sm_dbuf, sm);
849eda14cbcSMatt Macy sm->sm_dbuf = NULL;
850eda14cbcSMatt Macy sm->sm_phys = NULL;
851eda14cbcSMatt Macy
852eda14cbcSMatt Macy kmem_free(sm, sizeof (*sm));
853eda14cbcSMatt Macy }
854eda14cbcSMatt Macy
855eda14cbcSMatt Macy void
space_map_truncate(space_map_t * sm,int blocksize,dmu_tx_t * tx)856eda14cbcSMatt Macy space_map_truncate(space_map_t *sm, int blocksize, dmu_tx_t *tx)
857eda14cbcSMatt Macy {
858eda14cbcSMatt Macy objset_t *os = sm->sm_os;
859eda14cbcSMatt Macy spa_t *spa = dmu_objset_spa(os);
860eda14cbcSMatt Macy dmu_object_info_t doi;
861eda14cbcSMatt Macy
862eda14cbcSMatt Macy ASSERT(dsl_pool_sync_context(dmu_objset_pool(os)));
863eda14cbcSMatt Macy ASSERT(dmu_tx_is_syncing(tx));
864eda14cbcSMatt Macy VERIFY3U(dmu_tx_get_txg(tx), <=, spa_final_dirty_txg(spa));
865eda14cbcSMatt Macy
866eda14cbcSMatt Macy dmu_object_info_from_db(sm->sm_dbuf, &doi);
867eda14cbcSMatt Macy
868eda14cbcSMatt Macy /*
869eda14cbcSMatt Macy * If the space map has the wrong bonus size (because
870eda14cbcSMatt Macy * SPA_FEATURE_SPACEMAP_HISTOGRAM has recently been enabled), or
871eda14cbcSMatt Macy * the wrong block size (because space_map_blksz has changed),
872eda14cbcSMatt Macy * free and re-allocate its object with the updated sizes.
873eda14cbcSMatt Macy *
874eda14cbcSMatt Macy * Otherwise, just truncate the current object.
875eda14cbcSMatt Macy */
876eda14cbcSMatt Macy if ((spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM) &&
877eda14cbcSMatt Macy doi.doi_bonus_size != sizeof (space_map_phys_t)) ||
878eda14cbcSMatt Macy doi.doi_data_block_size != blocksize ||
879eda14cbcSMatt Macy doi.doi_metadata_block_size != 1 << space_map_ibs) {
880eda14cbcSMatt Macy zfs_dbgmsg("txg %llu, spa %s, sm %px, reallocating "
88133b8c039SMartin Matuska "object[%llu]: old bonus %llu, old blocksz %u",
88233b8c039SMartin Matuska (u_longlong_t)dmu_tx_get_txg(tx), spa_name(spa), sm,
88333b8c039SMartin Matuska (u_longlong_t)sm->sm_object,
88433b8c039SMartin Matuska (u_longlong_t)doi.doi_bonus_size,
88533b8c039SMartin Matuska doi.doi_data_block_size);
886eda14cbcSMatt Macy
887eda14cbcSMatt Macy space_map_free(sm, tx);
888eda14cbcSMatt Macy dmu_buf_rele(sm->sm_dbuf, sm);
889eda14cbcSMatt Macy
890eda14cbcSMatt Macy sm->sm_object = space_map_alloc(sm->sm_os, blocksize, tx);
891eda14cbcSMatt Macy VERIFY0(space_map_open_impl(sm));
892eda14cbcSMatt Macy } else {
893eda14cbcSMatt Macy VERIFY0(dmu_free_range(os, space_map_object(sm), 0, -1ULL, tx));
894eda14cbcSMatt Macy
895eda14cbcSMatt Macy /*
896eda14cbcSMatt Macy * If the spacemap is reallocated, its histogram
897eda14cbcSMatt Macy * will be reset. Do the same in the common case so that
898eda14cbcSMatt Macy * bugs related to the uncommon case do not go unnoticed.
899eda14cbcSMatt Macy */
900da5137abSMartin Matuska memset(sm->sm_phys->smp_histogram, 0,
901eda14cbcSMatt Macy sizeof (sm->sm_phys->smp_histogram));
902eda14cbcSMatt Macy }
903eda14cbcSMatt Macy
904eda14cbcSMatt Macy dmu_buf_will_dirty(sm->sm_dbuf, tx);
905eda14cbcSMatt Macy sm->sm_phys->smp_length = 0;
906eda14cbcSMatt Macy sm->sm_phys->smp_alloc = 0;
907eda14cbcSMatt Macy }
908eda14cbcSMatt Macy
909eda14cbcSMatt Macy uint64_t
space_map_alloc(objset_t * os,int blocksize,dmu_tx_t * tx)910eda14cbcSMatt Macy space_map_alloc(objset_t *os, int blocksize, dmu_tx_t *tx)
911eda14cbcSMatt Macy {
912eda14cbcSMatt Macy spa_t *spa = dmu_objset_spa(os);
913eda14cbcSMatt Macy uint64_t object;
914eda14cbcSMatt Macy int bonuslen;
915eda14cbcSMatt Macy
916eda14cbcSMatt Macy if (spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM)) {
917eda14cbcSMatt Macy spa_feature_incr(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM, tx);
918eda14cbcSMatt Macy bonuslen = sizeof (space_map_phys_t);
919eda14cbcSMatt Macy ASSERT3U(bonuslen, <=, dmu_bonus_max());
920eda14cbcSMatt Macy } else {
921eda14cbcSMatt Macy bonuslen = SPACE_MAP_SIZE_V0;
922eda14cbcSMatt Macy }
923eda14cbcSMatt Macy
924eda14cbcSMatt Macy object = dmu_object_alloc_ibs(os, DMU_OT_SPACE_MAP, blocksize,
925eda14cbcSMatt Macy space_map_ibs, DMU_OT_SPACE_MAP_HEADER, bonuslen, tx);
926eda14cbcSMatt Macy
927eda14cbcSMatt Macy return (object);
928eda14cbcSMatt Macy }
929eda14cbcSMatt Macy
930eda14cbcSMatt Macy void
space_map_free_obj(objset_t * os,uint64_t smobj,dmu_tx_t * tx)931eda14cbcSMatt Macy space_map_free_obj(objset_t *os, uint64_t smobj, dmu_tx_t *tx)
932eda14cbcSMatt Macy {
933eda14cbcSMatt Macy spa_t *spa = dmu_objset_spa(os);
934eda14cbcSMatt Macy if (spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM)) {
935eda14cbcSMatt Macy dmu_object_info_t doi;
936eda14cbcSMatt Macy
937eda14cbcSMatt Macy VERIFY0(dmu_object_info(os, smobj, &doi));
938eda14cbcSMatt Macy if (doi.doi_bonus_size != SPACE_MAP_SIZE_V0) {
939eda14cbcSMatt Macy spa_feature_decr(spa,
940eda14cbcSMatt Macy SPA_FEATURE_SPACEMAP_HISTOGRAM, tx);
941eda14cbcSMatt Macy }
942eda14cbcSMatt Macy }
943eda14cbcSMatt Macy
944eda14cbcSMatt Macy VERIFY0(dmu_object_free(os, smobj, tx));
945eda14cbcSMatt Macy }
946eda14cbcSMatt Macy
947eda14cbcSMatt Macy void
space_map_free(space_map_t * sm,dmu_tx_t * tx)948eda14cbcSMatt Macy space_map_free(space_map_t *sm, dmu_tx_t *tx)
949eda14cbcSMatt Macy {
950eda14cbcSMatt Macy if (sm == NULL)
951eda14cbcSMatt Macy return;
952eda14cbcSMatt Macy
953eda14cbcSMatt Macy space_map_free_obj(sm->sm_os, space_map_object(sm), tx);
954eda14cbcSMatt Macy sm->sm_object = 0;
955eda14cbcSMatt Macy }
956eda14cbcSMatt Macy
957eda14cbcSMatt Macy /*
958eda14cbcSMatt Macy * Given a range tree, it makes a worst-case estimate of how much
959eda14cbcSMatt Macy * space would the tree's segments take if they were written to
960eda14cbcSMatt Macy * the given space map.
961eda14cbcSMatt Macy */
962eda14cbcSMatt Macy uint64_t
space_map_estimate_optimal_size(space_map_t * sm,range_tree_t * rt,uint64_t vdev_id)963eda14cbcSMatt Macy space_map_estimate_optimal_size(space_map_t *sm, range_tree_t *rt,
964eda14cbcSMatt Macy uint64_t vdev_id)
965eda14cbcSMatt Macy {
966eda14cbcSMatt Macy spa_t *spa = dmu_objset_spa(sm->sm_os);
967eda14cbcSMatt Macy uint64_t shift = sm->sm_shift;
968eda14cbcSMatt Macy uint64_t *histogram = rt->rt_histogram;
969eda14cbcSMatt Macy uint64_t entries_for_seg = 0;
970eda14cbcSMatt Macy
971eda14cbcSMatt Macy /*
972eda14cbcSMatt Macy * In order to get a quick estimate of the optimal size that this
973eda14cbcSMatt Macy * range tree would have on-disk as a space map, we iterate through
974eda14cbcSMatt Macy * its histogram buckets instead of iterating through its nodes.
975eda14cbcSMatt Macy *
976eda14cbcSMatt Macy * Note that this is a highest-bound/worst-case estimate for the
977eda14cbcSMatt Macy * following reasons:
978eda14cbcSMatt Macy *
979eda14cbcSMatt Macy * 1] We assume that we always add a debug padding for each block
980eda14cbcSMatt Macy * we write and we also assume that we start at the last word
981eda14cbcSMatt Macy * of a block attempting to write a two-word entry.
982eda14cbcSMatt Macy * 2] Rounding up errors due to the way segments are distributed
983eda14cbcSMatt Macy * in the buckets of the range tree's histogram.
984eda14cbcSMatt Macy * 3] The activation of zfs_force_some_double_word_sm_entries
985eda14cbcSMatt Macy * (tunable) when testing.
986eda14cbcSMatt Macy *
987eda14cbcSMatt Macy * = Math and Rounding Errors =
988eda14cbcSMatt Macy *
989eda14cbcSMatt Macy * rt_histogram[i] bucket of a range tree represents the number
990eda14cbcSMatt Macy * of entries in [2^i, (2^(i+1))-1] of that range_tree. Given
991eda14cbcSMatt Macy * that, we want to divide the buckets into groups: Buckets that
992eda14cbcSMatt Macy * can be represented using a single-word entry, ones that can
993eda14cbcSMatt Macy * be represented with a double-word entry, and ones that can
994eda14cbcSMatt Macy * only be represented with multiple two-word entries.
995eda14cbcSMatt Macy *
996eda14cbcSMatt Macy * [Note that if the new encoding feature is not enabled there
997eda14cbcSMatt Macy * are only two groups: single-word entry buckets and multiple
998eda14cbcSMatt Macy * single-word entry buckets. The information below assumes
999eda14cbcSMatt Macy * two-word entries enabled, but it can easily applied when
1000eda14cbcSMatt Macy * the feature is not enabled]
1001eda14cbcSMatt Macy *
1002eda14cbcSMatt Macy * To find the highest bucket that can be represented with a
1003eda14cbcSMatt Macy * single-word entry we look at the maximum run that such entry
1004eda14cbcSMatt Macy * can have, which is 2^(SM_RUN_BITS + sm_shift) [remember that
1005eda14cbcSMatt Macy * the run of a space map entry is shifted by sm_shift, thus we
1006eda14cbcSMatt Macy * add it to the exponent]. This way, excluding the value of the
1007eda14cbcSMatt Macy * maximum run that can be represented by a single-word entry,
1008eda14cbcSMatt Macy * all runs that are smaller exist in buckets 0 to
1009eda14cbcSMatt Macy * SM_RUN_BITS + shift - 1.
1010eda14cbcSMatt Macy *
1011eda14cbcSMatt Macy * To find the highest bucket that can be represented with a
1012eda14cbcSMatt Macy * double-word entry, we follow the same approach. Finally, any
1013eda14cbcSMatt Macy * bucket higher than that are represented with multiple two-word
1014eda14cbcSMatt Macy * entries. To be more specific, if the highest bucket whose
1015eda14cbcSMatt Macy * segments can be represented with a single two-word entry is X,
1016eda14cbcSMatt Macy * then bucket X+1 will need 2 two-word entries for each of its
1017eda14cbcSMatt Macy * segments, X+2 will need 4, X+3 will need 8, ...etc.
1018eda14cbcSMatt Macy *
1019eda14cbcSMatt Macy * With all of the above we make our estimation based on bucket
1020eda14cbcSMatt Macy * groups. There is a rounding error though. As we mentioned in
1021eda14cbcSMatt Macy * the example with the one-word entry, the maximum run that can
1022eda14cbcSMatt Macy * be represented in a one-word entry 2^(SM_RUN_BITS + shift) is
1023eda14cbcSMatt Macy * not part of bucket SM_RUN_BITS + shift - 1. Thus, segments of
1024eda14cbcSMatt Macy * that length fall into the next bucket (and bucket group) where
1025eda14cbcSMatt Macy * we start counting two-word entries and this is one more reason
1026eda14cbcSMatt Macy * why the estimated size may end up being bigger than the actual
1027eda14cbcSMatt Macy * size written.
1028eda14cbcSMatt Macy */
1029eda14cbcSMatt Macy uint64_t size = 0;
1030eda14cbcSMatt Macy uint64_t idx = 0;
1031eda14cbcSMatt Macy
1032eda14cbcSMatt Macy if (!spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_V2) ||
1033eda14cbcSMatt Macy (vdev_id == SM_NO_VDEVID && sm->sm_size < SM_OFFSET_MAX)) {
1034eda14cbcSMatt Macy
1035eda14cbcSMatt Macy /*
1036eda14cbcSMatt Macy * If we are trying to force some double word entries just
1037eda14cbcSMatt Macy * assume the worst-case of every single word entry being
1038eda14cbcSMatt Macy * written as a double word entry.
1039eda14cbcSMatt Macy */
1040eda14cbcSMatt Macy uint64_t entry_size =
1041eda14cbcSMatt Macy (spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_V2) &&
1042eda14cbcSMatt Macy zfs_force_some_double_word_sm_entries) ?
1043eda14cbcSMatt Macy (2 * sizeof (uint64_t)) : sizeof (uint64_t);
1044eda14cbcSMatt Macy
1045eda14cbcSMatt Macy uint64_t single_entry_max_bucket = SM_RUN_BITS + shift - 1;
1046eda14cbcSMatt Macy for (; idx <= single_entry_max_bucket; idx++)
1047eda14cbcSMatt Macy size += histogram[idx] * entry_size;
1048eda14cbcSMatt Macy
1049eda14cbcSMatt Macy if (!spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_V2)) {
1050eda14cbcSMatt Macy for (; idx < RANGE_TREE_HISTOGRAM_SIZE; idx++) {
1051eda14cbcSMatt Macy ASSERT3U(idx, >=, single_entry_max_bucket);
1052eda14cbcSMatt Macy entries_for_seg =
1053eda14cbcSMatt Macy 1ULL << (idx - single_entry_max_bucket);
1054eda14cbcSMatt Macy size += histogram[idx] *
1055eda14cbcSMatt Macy entries_for_seg * entry_size;
1056eda14cbcSMatt Macy }
1057eda14cbcSMatt Macy return (size);
1058eda14cbcSMatt Macy }
1059eda14cbcSMatt Macy }
1060eda14cbcSMatt Macy
1061eda14cbcSMatt Macy ASSERT(spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_V2));
1062eda14cbcSMatt Macy
1063eda14cbcSMatt Macy uint64_t double_entry_max_bucket = SM2_RUN_BITS + shift - 1;
1064eda14cbcSMatt Macy for (; idx <= double_entry_max_bucket; idx++)
1065eda14cbcSMatt Macy size += histogram[idx] * 2 * sizeof (uint64_t);
1066eda14cbcSMatt Macy
1067eda14cbcSMatt Macy for (; idx < RANGE_TREE_HISTOGRAM_SIZE; idx++) {
1068eda14cbcSMatt Macy ASSERT3U(idx, >=, double_entry_max_bucket);
1069eda14cbcSMatt Macy entries_for_seg = 1ULL << (idx - double_entry_max_bucket);
1070eda14cbcSMatt Macy size += histogram[idx] *
1071eda14cbcSMatt Macy entries_for_seg * 2 * sizeof (uint64_t);
1072eda14cbcSMatt Macy }
1073eda14cbcSMatt Macy
1074eda14cbcSMatt Macy /*
1075eda14cbcSMatt Macy * Assume the worst case where we start with the padding at the end
1076eda14cbcSMatt Macy * of the current block and we add an extra padding entry at the end
1077eda14cbcSMatt Macy * of all subsequent blocks.
1078eda14cbcSMatt Macy */
1079eda14cbcSMatt Macy size += ((size / sm->sm_blksz) + 1) * sizeof (uint64_t);
1080eda14cbcSMatt Macy
1081eda14cbcSMatt Macy return (size);
1082eda14cbcSMatt Macy }
1083eda14cbcSMatt Macy
1084eda14cbcSMatt Macy uint64_t
space_map_object(space_map_t * sm)1085eda14cbcSMatt Macy space_map_object(space_map_t *sm)
1086eda14cbcSMatt Macy {
1087eda14cbcSMatt Macy return (sm != NULL ? sm->sm_object : 0);
1088eda14cbcSMatt Macy }
1089eda14cbcSMatt Macy
1090eda14cbcSMatt Macy int64_t
space_map_allocated(space_map_t * sm)1091eda14cbcSMatt Macy space_map_allocated(space_map_t *sm)
1092eda14cbcSMatt Macy {
1093eda14cbcSMatt Macy return (sm != NULL ? sm->sm_phys->smp_alloc : 0);
1094eda14cbcSMatt Macy }
1095eda14cbcSMatt Macy
1096eda14cbcSMatt Macy uint64_t
space_map_length(space_map_t * sm)1097eda14cbcSMatt Macy space_map_length(space_map_t *sm)
1098eda14cbcSMatt Macy {
1099eda14cbcSMatt Macy return (sm != NULL ? sm->sm_phys->smp_length : 0);
1100eda14cbcSMatt Macy }
1101eda14cbcSMatt Macy
1102eda14cbcSMatt Macy uint64_t
space_map_nblocks(space_map_t * sm)1103eda14cbcSMatt Macy space_map_nblocks(space_map_t *sm)
1104eda14cbcSMatt Macy {
1105eda14cbcSMatt Macy if (sm == NULL)
1106eda14cbcSMatt Macy return (0);
1107eda14cbcSMatt Macy return (DIV_ROUND_UP(space_map_length(sm), sm->sm_blksz));
1108eda14cbcSMatt Macy }
1109