Lines Matching defs:abd
109 abd_verify(abd_t *abd)
112 if (abd_is_from_pages(abd)) {
113 ASSERT3U(abd->abd_size, <=, DMU_MAX_ACCESS);
115 ASSERT3U(abd->abd_size, <=, SPA_MAXBLOCKSIZE);
117 ASSERT3U(abd->abd_flags, ==, abd->abd_flags & (ABD_FLAG_LINEAR |
121 IMPLY(abd->abd_parent != NULL, !(abd->abd_flags & ABD_FLAG_OWNER));
122 IMPLY(abd->abd_flags & ABD_FLAG_META, abd->abd_flags & ABD_FLAG_OWNER);
123 if (abd_is_linear(abd)) {
124 ASSERT3U(abd->abd_size, >, 0);
125 ASSERT3P(ABD_LINEAR_BUF(abd), !=, NULL);
126 } else if (abd_is_gang(abd)) {
128 for (abd_t *cabd = list_head(&ABD_GANG(abd).abd_gang_chain);
130 cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
135 ASSERT3U(abd->abd_size, ==, child_sizes);
137 ASSERT3U(abd->abd_size, >, 0);
138 abd_verify_scatter(abd);
144 abd_init_struct(abd_t *abd)
146 list_link_init(&abd->abd_gang_link);
147 mutex_init(&abd->abd_mtx, NULL, MUTEX_DEFAULT, NULL);
148 abd->abd_flags = 0;
150 zfs_refcount_create(&abd->abd_children);
151 abd->abd_parent = NULL;
153 abd->abd_size = 0;
157 abd_fini_struct(abd_t *abd)
159 mutex_destroy(&abd->abd_mtx);
160 ASSERT(!list_link_active(&abd->abd_gang_link));
162 zfs_refcount_destroy(&abd->abd_children);
169 abd_t *abd = abd_alloc_struct_impl(size);
170 abd_init_struct(abd);
171 abd->abd_flags |= ABD_FLAG_ALLOCD;
172 return (abd);
176 abd_free_struct(abd_t *abd)
178 abd_fini_struct(abd);
179 abd_free_struct_impl(abd);
194 abd_t *abd = abd_alloc_struct(size);
195 abd->abd_flags |= ABD_FLAG_OWNER;
196 abd->abd_u.abd_scatter.abd_offset = 0;
197 abd_alloc_chunks(abd, size);
200 abd->abd_flags |= ABD_FLAG_META;
202 abd->abd_size = size;
204 abd_update_scatter_stats(abd, ABDSTAT_INCR);
206 return (abd);
217 abd_t *abd = abd_alloc_struct(0);
221 abd->abd_flags |= ABD_FLAG_LINEAR | ABD_FLAG_OWNER;
223 abd->abd_flags |= ABD_FLAG_META;
225 abd->abd_size = size;
228 ABD_LINEAR_BUF(abd) = zio_buf_alloc(size);
230 ABD_LINEAR_BUF(abd) = zio_data_buf_alloc(size);
233 abd_update_linear_stats(abd, ABDSTAT_INCR);
235 return (abd);
239 abd_free_linear(abd_t *abd)
241 if (abd_is_linear_page(abd)) {
242 abd_free_linear_page(abd);
246 if (abd->abd_flags & ABD_FLAG_META) {
247 zio_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size);
249 zio_data_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size);
252 abd_update_linear_stats(abd, ABDSTAT_DECR);
256 abd_free_gang(abd_t *abd)
258 ASSERT(abd_is_gang(abd));
261 while ((cabd = list_head(&ABD_GANG(abd).abd_gang_chain)) != NULL) {
270 list_remove(&ABD_GANG(abd).abd_gang_chain, cabd);
275 list_destroy(&ABD_GANG(abd).abd_gang_chain);
279 abd_free_scatter(abd_t *abd)
281 abd_free_chunks(abd);
282 abd_update_scatter_stats(abd, ABDSTAT_DECR);
286 * Free an ABD. Use with any kind of abd: those created with abd_alloc_*()
297 abd_free(abd_t *abd)
299 if (abd == NULL)
302 abd_verify(abd);
304 IMPLY(abd->abd_flags & ABD_FLAG_OWNER, abd->abd_parent == NULL);
307 if (abd_is_gang(abd)) {
308 abd_free_gang(abd);
309 } else if (abd_is_linear(abd)) {
310 if (abd->abd_flags & ABD_FLAG_OWNER)
311 abd_free_linear(abd);
313 if (abd->abd_flags & ABD_FLAG_OWNER)
314 abd_free_scatter(abd);
318 if (abd->abd_parent != NULL) {
319 (void) zfs_refcount_remove_many(&abd->abd_parent->abd_children,
320 abd->abd_size, abd);
324 abd_fini_struct(abd);
325 if (abd->abd_flags & ABD_FLAG_ALLOCD)
326 abd_free_struct_impl(abd);
348 * IO's. To free this abd, abd_free() must be called.
353 abd_t *abd = abd_alloc_struct(0);
354 abd->abd_flags |= ABD_FLAG_GANG | ABD_FLAG_OWNER;
355 list_create(&ABD_GANG(abd).abd_gang_chain,
357 return (abd);
489 abd_gang_get_offset(abd_t *abd, size_t *off)
493 ASSERT(abd_is_gang(abd));
494 ASSERT3U(*off, <, abd->abd_size);
495 for (cabd = list_head(&ABD_GANG(abd).abd_gang_chain); cabd != NULL;
496 cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
513 abd_get_offset_impl(abd_t *abd, abd_t *sabd, size_t off, size_t size)
519 if (abd == NULL)
520 abd = abd_alloc_struct(0);
526 abd->abd_flags |= ABD_FLAG_LINEAR;
531 * that here for abd. This is required because we have to be
539 abd->abd_flags |= ABD_FLAG_FROM_PAGES |
543 ABD_LINEAR_BUF(abd) = (char *)ABD_LINEAR_BUF(sabd) + off;
546 if (abd == NULL) {
547 abd = abd_alloc_gang();
549 abd->abd_flags |= ABD_FLAG_GANG;
550 list_create(&ABD_GANG(abd).abd_gang_chain,
554 abd->abd_flags &= ~ABD_FLAG_OWNER;
561 abd_gang_add(abd, nabd, B_TRUE);
567 abd = abd_get_offset_scatter(abd, sabd, off, size);
570 ASSERT3P(abd, !=, NULL);
571 abd->abd_size = size;
573 abd->abd_parent = sabd;
574 (void) zfs_refcount_add_many(&sabd->abd_children, abd->abd_size, abd);
576 return (abd);
582 * of allocating memory for the abd_t struct, and updating the abd stats.
583 * Usually, the provided abd is returned, but in some circumstances (FreeBSD,
589 abd_get_offset_struct(abd_t *abd, abd_t *sabd, size_t off, size_t size)
592 abd_init_struct(abd);
593 result = abd_get_offset_impl(abd, sabd, off, size);
594 if (result != abd)
595 abd_fini_struct(abd);
629 abd_get_from_buf_impl(abd_t *abd, void *buf, size_t size)
638 abd->abd_flags |= ABD_FLAG_LINEAR;
639 abd->abd_size = size;
641 ABD_LINEAR_BUF(abd) = buf;
643 return (abd);
649 abd_t *abd = abd_alloc_struct(0);
650 return (abd_get_from_buf_impl(abd, buf, size));
654 abd_get_from_buf_struct(abd_t *abd, void *buf, size_t size)
656 abd_init_struct(abd);
657 return (abd_get_from_buf_impl(abd, buf, size));
664 abd_to_buf(abd_t *abd)
666 ASSERT(abd_is_linear(abd));
667 abd_verify(abd);
668 return (ABD_LINEAR_BUF(abd));
672 abd_release_ownership_of_buf(abd_t *abd)
674 ASSERT(abd_is_linear(abd));
675 ASSERT(abd->abd_flags & ABD_FLAG_OWNER);
684 ASSERT(!abd_is_linear_page(abd));
686 abd_verify(abd);
688 abd->abd_flags &= ~ABD_FLAG_OWNER;
690 abd->abd_flags &= ~ABD_FLAG_META;
692 abd_update_linear_stats(abd, ABDSTAT_DECR);
703 abd_take_ownership_of_buf(abd_t *abd, boolean_t is_metadata)
705 ASSERT(abd_is_linear(abd));
706 ASSERT(!(abd->abd_flags & ABD_FLAG_OWNER));
707 abd_verify(abd);
709 abd->abd_flags |= ABD_FLAG_OWNER;
711 abd->abd_flags |= ABD_FLAG_META;
714 abd_update_linear_stats(abd, ABDSTAT_INCR);
718 * Initializes an abd_iter based on whether the abd is a gang ABD
722 abd_init_abd_iter(abd_t *abd, struct abd_iter *aiter, size_t off)
726 if (abd_is_gang(abd)) {
727 cabd = abd_gang_get_offset(abd, &off);
733 abd_iter_init(aiter, abd);
745 abd_advance_abd_iter(abd_t *abd, abd_t *cabd, struct abd_iter *aiter,
749 if (abd_is_gang(abd) && abd_iter_at_end(aiter)) {
751 cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd);
761 abd_iterate_func(abd_t *abd, size_t off, size_t size,
770 abd_verify(abd);
771 ASSERT3U(off + size, <=, abd->abd_size);
773 abd_t *c_abd = abd_init_abd_iter(abd, &aiter, off);
776 IMPLY(abd_is_gang(abd), c_abd != NULL);
791 c_abd = abd_advance_abd_iter(abd, c_abd, &aiter, len);
799 abd_iterate_page_func(abd_t *abd, size_t off, size_t size,
808 abd_verify(abd);
809 ASSERT3U(off + size, <=, abd->abd_size);
811 abd_t *c_abd = abd_init_abd_iter(abd, &aiter, off);
814 IMPLY(abd_is_gang(abd), c_abd != NULL);
832 c_abd = abd_advance_abd_iter(abd, c_abd, &aiter, len);
855 * Copy abd to buf. (off is the offset in abd.)
858 abd_copy_to_buf_off(void *buf, abd_t *abd, size_t off, size_t size)
862 (void) abd_iterate_func(abd, off, size, abd_copy_to_buf_off_cb,
879 * Compare the contents of abd to buf. (off is the offset in abd.)
882 abd_cmp_buf_off(abd_t *abd, const void *buf, size_t off, size_t size)
886 return (abd_iterate_func(abd, off, size, abd_cmp_buf_off_cb, &ba_ptr));
901 * Copy from buf to abd. (off is the offset in abd.)
904 abd_copy_from_buf_off(abd_t *abd, const void *buf, size_t off, size_t size)
908 (void) abd_iterate_func(abd, off, size, abd_copy_from_buf_off_cb,
921 * Zero out the abd from a particular offset to the end.
924 abd_zero_off(abd_t *abd, size_t off, size_t size)
926 (void) abd_iterate_func(abd, off, size, abd_zero_off_cb, NULL);
1041 abd_cmp_zero_off(abd_t *abd, size_t off, size_t size)
1043 return (abd_iterate_func(abd, off, size, abd_cmp_zero_off_cb, NULL));