Lines Matching +full:non +full:- +full:linear
9 * or https://opensource.org/licenses/CDDL-1.0.
32 * (a) Linear buffer. In this case, all the data in the ABD is stored in one
35 * +-------------------+
36 * | ABD (linear) |
38 * | abd_size = ... | +--------------------------------+
39 * | abd_buf ------------->| raw buffer of size abd_size |
40 * +-------------------+ +--------------------------------+
44 * equal-sized chunks (from the abd_chunk_cache kmem_cache), with pointers
47 * +-------------------+
51 * | abd_offset = 0 | +-----------+
52 * | abd_chunks[0] ----------------------------->| chunk 0 |
53 * | abd_chunks[1] ---------------------+ +-----------+
54 * | ... | | +-----------+
55 * | abd_chunks[N-1] ---------+ +------->| chunk 1 |
56 * +-------------------+ | +-----------+
58 * | +-----------+
59 * +----------------->| chunk N-1 |
60 * +-----------+
62 * In addition to directly allocating a linear or scattered ABD, it is also
63 * possible to create an ABD by requesting the "sub-ABD" starting at an offset
64 * within an existing ABD. In linear buffers this is simple (set abd_buf of
68 * provide arbitrary rather than only chunk-aligned starting offsets, it also
70 * within the first chunk in abd_chunks. For both linear and scattered ABDs,
75 * Most consumers should never need to know what type of ABD they're using --
77 * using a linear ABD to a scattered one when doing so would be beneficial.
79 * If you need to use the data within an ABD directly, if you know it's linear
91 * As an additional feature, linear and scatter ABD's can be stitched together
95 * It is possible to make all ABDs linear by setting zfs_abd_scatter_enabled to
113 ASSERT3U(abd->abd_size, <=, DMU_MAX_ACCESS);
115 ASSERT3U(abd->abd_size, <=, SPA_MAXBLOCKSIZE);
117 ASSERT3U(abd->abd_flags, ==, abd->abd_flags & (ABD_FLAG_LINEAR |
121 IMPLY(abd->abd_parent != NULL, !(abd->abd_flags & ABD_FLAG_OWNER));
122 IMPLY(abd->abd_flags & ABD_FLAG_META, abd->abd_flags & ABD_FLAG_OWNER);
124 ASSERT3U(abd->abd_size, >, 0);
131 ASSERT(list_link_active(&cabd->abd_gang_link));
132 child_sizes += cabd->abd_size;
135 ASSERT3U(abd->abd_size, ==, child_sizes);
137 ASSERT3U(abd->abd_size, >, 0);
146 list_link_init(&abd->abd_gang_link);
147 mutex_init(&abd->abd_mtx, NULL, MUTEX_DEFAULT, NULL);
148 abd->abd_flags = 0;
150 zfs_refcount_create(&abd->abd_children);
151 abd->abd_parent = NULL;
153 abd->abd_size = 0;
159 mutex_destroy(&abd->abd_mtx);
160 ASSERT(!list_link_active(&abd->abd_gang_link));
162 zfs_refcount_destroy(&abd->abd_children);
171 abd->abd_flags |= ABD_FLAG_ALLOCD;
184 * don't care whether the ABD is linear or not.
195 abd->abd_flags |= ABD_FLAG_OWNER;
196 abd->abd_u.abd_scatter.abd_offset = 0;
200 abd->abd_flags |= ABD_FLAG_META;
202 abd->abd_size = size;
210 * Allocate an ABD that must be linear, along with its own underlying data
221 abd->abd_flags |= ABD_FLAG_LINEAR | ABD_FLAG_OWNER;
223 abd->abd_flags |= ABD_FLAG_META;
225 abd->abd_size = size;
246 if (abd->abd_flags & ABD_FLAG_META) {
247 zio_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size);
249 zio_data_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size);
268 mutex_enter(&cabd->abd_mtx);
269 ASSERT(list_link_active(&cabd->abd_gang_link));
271 mutex_exit(&cabd->abd_mtx);
272 if (cabd->abd_flags & ABD_FLAG_GANG_FREE)
290 * (scatterlist or linear buffer) will also be freed. (Subject to ownership
304 IMPLY(abd->abd_flags & ABD_FLAG_OWNER, abd->abd_parent == NULL);
310 if (abd->abd_flags & ABD_FLAG_OWNER)
313 if (abd->abd_flags & ABD_FLAG_OWNER)
318 if (abd->abd_parent != NULL) {
319 (void) zfs_refcount_remove_many(&abd->abd_parent->abd_children,
320 abd->abd_size, abd);
325 if (abd->abd_flags & ABD_FLAG_ALLOCD)
336 boolean_t is_metadata = (sabd->abd_flags & ABD_FLAG_META) != 0;
354 abd->abd_flags |= ABD_FLAG_GANG | ABD_FLAG_OWNER;
383 if (cabd->abd_parent != NULL) {
385 &cabd->abd_parent->abd_children,
386 cabd->abd_size, cabd);
387 cabd->abd_parent = NULL;
390 pabd->abd_size += cabd->abd_size;
391 cabd->abd_size = 0;
428 ASSERT(!list_link_active(&cabd->abd_gang_link));
441 mutex_enter(&cabd->abd_mtx);
442 if (list_link_active(&cabd->abd_gang_link)) {
471 child_abd->abd_flags |= ABD_FLAG_GANG_FREE;
475 child_abd->abd_flags |= ABD_FLAG_GANG_FREE;
480 mutex_exit(&cabd->abd_mtx);
481 pabd->abd_size += child_abd->abd_size;
494 ASSERT3U(*off, <, abd->abd_size);
497 if (*off >= cabd->abd_size)
498 *off -= cabd->abd_size;
507 * Allocate a new ABD, using the provided struct (if non-NULL, and if
508 * circumstances allow - otherwise allocate the struct). The returned ABD will
516 ASSERT3U(off + size, <=, sabd->abd_size);
526 abd->abd_flags |= ABD_FLAG_LINEAR;
539 abd->abd_flags |= ABD_FLAG_FROM_PAGES |
549 abd->abd_flags |= ABD_FLAG_GANG;
554 abd->abd_flags &= ~ABD_FLAG_OWNER;
558 int csize = MIN(left, cabd->abd_size - off);
562 left -= csize;
571 abd->abd_size = size;
573 abd->abd_parent = sabd;
574 (void) zfs_refcount_add_many(&sabd->abd_children, abd->abd_size, abd);
602 size_t size = sabd->abd_size > off ? sabd->abd_size - off : 0;
610 ASSERT3U(off + size, <=, sabd->abd_size);
626 * Create a linear ABD for an existing buf.
638 abd->abd_flags |= ABD_FLAG_LINEAR;
639 abd->abd_size = size;
661 * Get the raw buffer associated with a linear ABD.
675 ASSERT(abd->abd_flags & ABD_FLAG_OWNER);
680 * abd_release_ownership_of_buf() -> abd_get_from_buf() ->
682 * these "linear but not zio_[data_]buf_alloc()'ed" ABD's.
688 abd->abd_flags &= ~ABD_FLAG_OWNER;
690 abd->abd_flags &= ~ABD_FLAG_META;
698 * linear ABDs which were allocated via abd_get_from_buf(), or ones allocated
706 ASSERT(!(abd->abd_flags & ABD_FLAG_OWNER));
709 abd->abd_flags |= ABD_FLAG_OWNER;
711 abd->abd_flags |= ABD_FLAG_META;
771 ASSERT3U(off + size, <=, abd->abd_size);
790 size -= len;
809 ASSERT3U(off + size, <=, abd->abd_size);
831 size -= len;
848 (void) memcpy(ba_ptr->arg_buf, buf, size);
849 ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size;
872 ret = memcmp(buf, ba_ptr->arg_buf, size);
873 ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size;
894 (void) memcpy(buf, ba_ptr->arg_buf, size);
895 ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size;
931 * equal-sized chunks (passed to func as raw buffers). func could be called many
948 ASSERT3U(doff + size, <=, dabd->abd_size);
949 ASSERT3U(soff + size, <=, sabd->abd_size);
975 size -= len;
1016 ASSERT3U(dabd->abd_size, ==, sabd->abd_size);
1017 return (abd_iterate_func2(dabd, sabd, 0, 0, dabd->abd_size,
1022 * Check if ABD content is all-zeroes.
1052 * is the same when taking linear and when taking scatter
1071 ASSERT3U(off + csize, <=, cabds[i]->abd_size);
1078 ASSERT3U(off + dsize, <=, dabd->abd_size);
1113 for (i = parity-1; i >= 0; i--) {
1125 dsize -= dlen;
1128 csize -= len;
1164 ASSERT3U(tsize, <=, cabds[i]->abd_size);
1165 ASSERT3U(tsize, <=, tabds[i]->abd_size);
1196 for (i = parity-1; i >= 0; i--) {
1207 tsize -= len;