Lines Matching defs:rr
381 vdev_raidz_row_free(raidz_row_t *rr)
383 for (int c = 0; c < rr->rr_cols; c++) {
384 raidz_col_t *rc = &rr->rr_col[c];
392 if (rr->rr_abd_empty != NULL)
393 abd_free(rr->rr_abd_empty);
395 kmem_free(rr, offsetof(raidz_row_t, rr_col[rr->rr_scols]));
442 raidz_row_t *rr =
445 rr->rr_cols = cols;
446 rr->rr_scols = cols;
449 raidz_col_t *rc = &rr->rr_col[c];
464 return (rr);
473 raidz_row_t *rr = rm->rm_row[0];
482 if (rm->rm_skipstart < rr->rr_firstdatacol) {
485 } else if (rr->rr_scols < (rm->rm_skipstart + rm->rm_nskip)) {
487 (rm->rm_skipstart + rm->rm_nskip) % rr->rr_scols;
494 int skipped = rr->rr_scols - rr->rr_cols;
497 for (c = 0; c < rr->rr_firstdatacol; c++) {
498 raidz_col_t *rc = &rr->rr_col[c];
519 for (off = 0; c < rr->rr_cols; c++) {
520 raidz_col_t *rc = &rr->rr_col[c];
558 raidz_row_t *rr = rm->rm_row[0];
563 for (c = 0; c < rr->rr_firstdatacol; c++)
564 rr->rr_col[c].rc_abd =
565 abd_alloc_linear(rr->rr_col[c].rc_size, B_FALSE);
567 for (uint64_t off = 0; c < rr->rr_cols; c++) {
568 raidz_col_t *rc = &rr->rr_col[c];
586 raidz_row_t *rr;
636 rr = vdev_raidz_row_alloc(scols, zio);
637 rm->rm_row[0] = rr;
638 rr->rr_cols = acols;
639 rr->rr_bigcols = bc;
640 rr->rr_firstdatacol = nparity;
642 rr->rr_offset = zio->io_offset;
643 rr->rr_size = zio->io_size;
649 raidz_col_t *rc = &rr->rr_col[c];
693 ASSERT(rr->rr_cols >= 2);
694 ASSERT(rr->rr_col[0].rc_size == rr->rr_col[1].rc_size);
696 if (rr->rr_firstdatacol == 1 && (zio->io_offset & (1ULL << 20))) {
697 uint64_t devidx = rr->rr_col[0].rc_devidx;
698 o = rr->rr_col[0].rc_offset;
699 rr->rr_col[0].rc_devidx = rr->rr_col[1].rc_devidx;
700 rr->rr_col[0].rc_offset = rr->rr_col[1].rc_offset;
701 rr->rr_col[1].rc_devidx = devidx;
702 rr->rr_col[1].rc_offset = o;
782 raidz_row_t *rr = vdev_raidz_row_alloc(cols, zio);
783 rm->rm_row[row] = rr;
826 rr->rr_firstdatacol = nparity;
831 rr->rr_offset = b << ashift;
832 rr->rr_size = (rr->rr_cols - rr->rr_firstdatacol) << ashift;
835 for (int c = 0; c < rr->rr_cols; c++, child_id++) {
840 raidz_col_t *rc = &rr->rr_col[c];
855 uint64_t dc = c - rr->rr_firstdatacol;
856 if (c < rr->rr_firstdatacol) {
922 if (rr->rr_firstdatacol == 1 && rr->rr_cols > 1 &&
924 ASSERT(rr->rr_cols >= 2);
925 ASSERT(rr->rr_col[0].rc_size == rr->rr_col[1].rc_size);
927 int devidx0 = rr->rr_col[0].rc_devidx;
928 uint64_t offset0 = rr->rr_col[0].rc_offset;
929 int shadow_devidx0 = rr->rr_col[0].rc_shadow_devidx;
931 rr->rr_col[0].rc_shadow_offset;
933 rr->rr_col[0].rc_devidx = rr->rr_col[1].rc_devidx;
934 rr->rr_col[0].rc_offset = rr->rr_col[1].rc_offset;
935 rr->rr_col[0].rc_shadow_devidx =
936 rr->rr_col[1].rc_shadow_devidx;
937 rr->rr_col[0].rc_shadow_offset =
938 rr->rr_col[1].rc_shadow_offset;
940 rr->rr_col[1].rc_devidx = devidx0;
941 rr->rr_col[1].rc_offset = offset0;
942 rr->rr_col[1].rc_shadow_devidx = shadow_devidx0;
943 rr->rr_col[1].rc_shadow_offset = shadow_offset0;
964 raidz_row_t *rr = rm->rm_row[i];
965 for (int c = 0; c < rr->rr_cols; c++) {
966 raidz_col_t *rc = &rr->rr_col[c];
1018 raidz_row_t *rr = rm->rm_row[i];
1019 for (int c = 0; c < rr->rr_firstdatacol; c++) {
1020 raidz_col_t *rc = &rr->rr_col[c];
1035 raidz_row_t *rr = rm->rm_row[i];
1036 for (int c = 0; c < rr->rr_firstdatacol; c++) {
1037 raidz_col_t *rc = &rr->rr_col[c];
1112 vdev_raidz_generate_parity_p(raidz_row_t *rr)
1114 uint64_t *p = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd);
1116 for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
1117 abd_t *src = rr->rr_col[c].rc_abd;
1119 if (c == rr->rr_firstdatacol) {
1120 abd_copy_to_buf(p, src, rr->rr_col[c].rc_size);
1123 (void) abd_iterate_func(src, 0, rr->rr_col[c].rc_size,
1130 vdev_raidz_generate_parity_pq(raidz_row_t *rr)
1132 uint64_t *p = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd);
1133 uint64_t *q = abd_to_buf(rr->rr_col[VDEV_RAIDZ_Q].rc_abd);
1134 uint64_t pcnt = rr->rr_col[VDEV_RAIDZ_P].rc_size / sizeof (p[0]);
1135 ASSERT(rr->rr_col[VDEV_RAIDZ_P].rc_size ==
1136 rr->rr_col[VDEV_RAIDZ_Q].rc_size);
1138 for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
1139 abd_t *src = rr->rr_col[c].rc_abd;
1141 uint64_t ccnt = rr->rr_col[c].rc_size / sizeof (p[0]);
1143 if (c == rr->rr_firstdatacol) {
1145 abd_copy_to_buf(p, src, rr->rr_col[c].rc_size);
1146 (void) memcpy(q, p, rr->rr_col[c].rc_size);
1156 (void) abd_iterate_func(src, 0, rr->rr_col[c].rc_size,
1172 vdev_raidz_generate_parity_pqr(raidz_row_t *rr)
1174 uint64_t *p = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd);
1175 uint64_t *q = abd_to_buf(rr->rr_col[VDEV_RAIDZ_Q].rc_abd);
1176 uint64_t *r = abd_to_buf(rr->rr_col[VDEV_RAIDZ_R].rc_abd);
1177 uint64_t pcnt = rr->rr_col[VDEV_RAIDZ_P].rc_size / sizeof (p[0]);
1178 ASSERT(rr->rr_col[VDEV_RAIDZ_P].rc_size ==
1179 rr->rr_col[VDEV_RAIDZ_Q].rc_size);
1180 ASSERT(rr->rr_col[VDEV_RAIDZ_P].rc_size ==
1181 rr->rr_col[VDEV_RAIDZ_R].rc_size);
1183 for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
1184 abd_t *src = rr->rr_col[c].rc_abd;
1186 uint64_t ccnt = rr->rr_col[c].rc_size / sizeof (p[0]);
1188 if (c == rr->rr_firstdatacol) {
1190 abd_copy_to_buf(p, src, rr->rr_col[c].rc_size);
1191 (void) memcpy(q, p, rr->rr_col[c].rc_size);
1192 (void) memcpy(r, p, rr->rr_col[c].rc_size);
1203 (void) abd_iterate_func(src, 0, rr->rr_col[c].rc_size,
1224 vdev_raidz_generate_parity_row(raidz_map_t *rm, raidz_row_t *rr)
1226 if (rr->rr_cols == 0) {
1237 if (vdev_raidz_math_generate(rm, rr) != RAIDZ_ORIGINAL_IMPL)
1240 switch (rr->rr_firstdatacol) {
1242 vdev_raidz_generate_parity_p(rr);
1245 vdev_raidz_generate_parity_pq(rr);
1248 vdev_raidz_generate_parity_pqr(rr);
1259 raidz_row_t *rr = rm->rm_row[i];
1260 vdev_raidz_generate_parity_row(rm, rr);
1381 vdev_raidz_reconstruct_p(raidz_row_t *rr, int *tgts, int ntgts)
1387 zfs_dbgmsg("reconstruct_p(rm=%px x=%u)", rr, x);
1390 ASSERT3U(x, >=, rr->rr_firstdatacol);
1391 ASSERT3U(x, <, rr->rr_cols);
1393 ASSERT3U(rr->rr_col[x].rc_size, <=, rr->rr_col[VDEV_RAIDZ_P].rc_size);
1395 src = rr->rr_col[VDEV_RAIDZ_P].rc_abd;
1396 dst = rr->rr_col[x].rc_abd;
1398 abd_copy_from_buf(dst, abd_to_buf(src), rr->rr_col[x].rc_size);
1400 for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
1401 uint64_t size = MIN(rr->rr_col[x].rc_size,
1402 rr->rr_col[c].rc_size);
1404 src = rr->rr_col[c].rc_abd;
1415 vdev_raidz_reconstruct_q(raidz_row_t *rr, int *tgts, int ntgts)
1422 zfs_dbgmsg("reconstruct_q(rm=%px x=%u)", rr, x);
1426 ASSERT(rr->rr_col[x].rc_size <= rr->rr_col[VDEV_RAIDZ_Q].rc_size);
1428 for (c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
1429 uint64_t size = (c == x) ? 0 : MIN(rr->rr_col[x].rc_size,
1430 rr->rr_col[c].rc_size);
1432 src = rr->rr_col[c].rc_abd;
1433 dst = rr->rr_col[x].rc_abd;
1435 if (c == rr->rr_firstdatacol) {
1437 if (rr->rr_col[x].rc_size > size) {
1439 rr->rr_col[x].rc_size - size);
1442 ASSERT3U(size, <=, rr->rr_col[x].rc_size);
1446 size, rr->rr_col[x].rc_size - size,
1451 src = rr->rr_col[VDEV_RAIDZ_Q].rc_abd;
1452 dst = rr->rr_col[x].rc_abd;
1453 exp = 255 - (rr->rr_cols - 1 - x);
1456 (void) abd_iterate_func(dst, 0, rr->rr_col[x].rc_size,
1461 vdev_raidz_reconstruct_pq(raidz_row_t *rr, int *tgts, int ntgts)
1471 zfs_dbgmsg("reconstruct_pq(rm=%px x=%u y=%u)", rr, x, y);
1475 ASSERT(x >= rr->rr_firstdatacol);
1476 ASSERT(y < rr->rr_cols);
1478 ASSERT(rr->rr_col[x].rc_size >= rr->rr_col[y].rc_size);
1487 pdata = rr->rr_col[VDEV_RAIDZ_P].rc_abd;
1488 qdata = rr->rr_col[VDEV_RAIDZ_Q].rc_abd;
1489 xsize = rr->rr_col[x].rc_size;
1490 ysize = rr->rr_col[y].rc_size;
1492 rr->rr_col[VDEV_RAIDZ_P].rc_abd =
1493 abd_alloc_linear(rr->rr_col[VDEV_RAIDZ_P].rc_size, B_TRUE);
1494 rr->rr_col[VDEV_RAIDZ_Q].rc_abd =
1495 abd_alloc_linear(rr->rr_col[VDEV_RAIDZ_Q].rc_size, B_TRUE);
1496 rr->rr_col[x].rc_size = 0;
1497 rr->rr_col[y].rc_size = 0;
1499 vdev_raidz_generate_parity_pq(rr);
1501 rr->rr_col[x].rc_size = xsize;
1502 rr->rr_col[y].rc_size = ysize;
1506 pxy = abd_to_buf(rr->rr_col[VDEV_RAIDZ_P].rc_abd);
1507 qxy = abd_to_buf(rr->rr_col[VDEV_RAIDZ_Q].rc_abd);
1508 xd = rr->rr_col[x].rc_abd;
1509 yd = rr->rr_col[y].rc_abd;
1527 b = vdev_raidz_pow2[255 - (rr->rr_cols - 1 - x)];
1541 abd_free(rr->rr_col[VDEV_RAIDZ_P].rc_abd);
1542 abd_free(rr->rr_col[VDEV_RAIDZ_Q].rc_abd);
1547 rr->rr_col[VDEV_RAIDZ_P].rc_abd = pdata;
1548 rr->rr_col[VDEV_RAIDZ_Q].rc_abd = qdata;
1704 vdev_raidz_matrix_init(raidz_row_t *rr, int n, int nmap, int *map,
1710 ASSERT(n == rr->rr_cols - rr->rr_firstdatacol);
1734 vdev_raidz_matrix_invert(raidz_row_t *rr, int n, int nmissing, int *missing,
1746 ASSERT3S(used[i], <, rr->rr_firstdatacol);
1749 ASSERT3S(used[i], >=, rr->rr_firstdatacol);
1766 ASSERT3U(used[j], >=, rr->rr_firstdatacol);
1767 jj = used[j] - rr->rr_firstdatacol;
1828 vdev_raidz_matrix_reconstruct(raidz_row_t *rr, int n, int nmissing,
1860 ASSERT3U(c, <, rr->rr_cols);
1862 ccount = rr->rr_col[c].rc_size;
1863 ASSERT(ccount >= rr->rr_col[missing[0]].rc_size || i > 0);
1866 src = abd_to_buf(rr->rr_col[c].rc_abd);
1868 cc = missing[j] + rr->rr_firstdatacol;
1869 ASSERT3U(cc, >=, rr->rr_firstdatacol);
1870 ASSERT3U(cc, <, rr->rr_cols);
1873 dcount[j] = rr->rr_col[cc].rc_size;
1875 dst[j] = abd_to_buf(rr->rr_col[cc].rc_abd);
1906 vdev_raidz_reconstruct_general(raidz_row_t *rr, int *tgts, int ntgts)
1922 zfs_dbgmsg("reconstruct_general(rm=%px ntgts=%u)", rr, ntgts);
1927 for (i = rr->rr_firstdatacol; i < rr->rr_cols; i++) {
1928 ASSERT(rr->rr_col[i].rc_abd != NULL);
1929 if (!abd_is_linear(rr->rr_col[i].rc_abd)) {
1930 bufs = kmem_alloc(rr->rr_cols * sizeof (abd_t *),
1933 for (c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
1934 raidz_col_t *col = &rr->rr_col[c];
1949 n = rr->rr_cols - rr->rr_firstdatacol;
1956 if (tgts[t] >= rr->rr_firstdatacol) {
1958 tgts[t] - rr->rr_firstdatacol;
1968 ASSERT(c < rr->rr_firstdatacol);
1998 for (tt = 0, c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
2000 c == missing_rows[tt] + rr->rr_firstdatacol) {
2013 vdev_raidz_matrix_init(rr, n, nmissing_rows, parity_map, rows);
2018 vdev_raidz_matrix_invert(rr, n, nmissing_rows, missing_rows, rows,
2024 vdev_raidz_matrix_reconstruct(rr, n, nmissing_rows, missing_rows,
2033 for (c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
2034 raidz_col_t *col = &rr->rr_col[c];
2042 kmem_free(bufs, rr->rr_cols * sizeof (abd_t *));
2047 vdev_raidz_reconstruct_row(raidz_map_t *rm, raidz_row_t *rr,
2058 rr, nt, (int)rr->rr_cols, (int)rr->rr_missingdata,
2059 (int)rr->rr_missingparity);
2062 nbadparity = rr->rr_firstdatacol;
2063 nbaddata = rr->rr_cols - nbadparity;
2065 for (i = 0, c = 0; c < rr->rr_cols; c++) {
2069 rr, c, (int)rr->rr_col[c].rc_devidx,
2070 (long long)rr->rr_col[c].rc_offset,
2071 (int)rr->rr_col[c].rc_error);
2073 if (c < rr->rr_firstdatacol)
2079 } else if (rr->rr_col[c].rc_error != 0) {
2081 } else if (c >= rr->rr_firstdatacol) {
2096 ret = vdev_raidz_math_reconstruct(rm, rr, parity_valid, dt, nbaddata);
2106 vdev_raidz_reconstruct_p(rr, dt, 1);
2110 ASSERT(rr->rr_firstdatacol > 1);
2113 vdev_raidz_reconstruct_q(rr, dt, 1);
2117 ASSERT(rr->rr_firstdatacol > 2);
2121 ASSERT(rr->rr_firstdatacol > 1);
2125 vdev_raidz_reconstruct_pq(rr, dt, 2);
2129 ASSERT(rr->rr_firstdatacol > 2);
2134 vdev_raidz_reconstruct_general(rr, tgts, ntgts);
2304 vdev_raidz_io_verify(zio_t *zio, raidz_map_t *rm, raidz_row_t *rr, int col)
2309 logical_rs.rs_start = rr->rr_offset;
2311 vdev_raidz_asize(zio->io_vd, rr->rr_size,
2314 raidz_col_t *rc = &rr->rr_col[col];
2345 vdev_raidz_io_start_write(zio_t *zio, raidz_row_t *rr)
2350 vdev_raidz_generate_parity_row(rm, rr);
2352 for (int c = 0; c < rr->rr_scols; c++) {
2353 raidz_col_t *rc = &rr->rr_col[c];
2357 vdev_raidz_io_verify(zio, rm, rr, c);
2398 raidz_row_t *rr = rm->rm_row[0];
2399 for (int c = 0; c < rr->rr_scols; c++) {
2400 raidz_col_t *rc = &rr->rr_col[c];
2416 vdev_raidz_io_start_read_row(zio_t *zio, raidz_row_t *rr, boolean_t forceparity)
2424 for (int c = rr->rr_cols - 1; c >= 0; c--) {
2425 raidz_col_t *rc = &rr->rr_col[c];
2430 if (c >= rr->rr_firstdatacol)
2431 rr->rr_missingdata++;
2433 rr->rr_missingparity++;
2440 if (c >= rr->rr_firstdatacol)
2441 rr->rr_missingdata++;
2443 rr->rr_missingparity++;
2449 c >= rr->rr_firstdatacol || rr->rr_missingdata > 0 ||
2503 raidz_row_t *rr = rm->rm_row[i];
2504 vdev_raidz_io_start_read_row(zio, rr, forceparity);
2677 raidz_parity_verify(zio_t *zio, raidz_row_t *rr)
2691 for (c = 0; c < rr->rr_firstdatacol; c++) {
2692 rc = &rr->rr_col[c];
2705 if (rr->rr_nempty && rr->rr_abd_empty != NULL)
2706 ret += vdev_draid_map_verify_empty(zio, rr);
2713 vdev_raidz_generate_parity_row(rm, rr);
2715 for (c = 0; c < rr->rr_firstdatacol; c++) {
2716 rc = &rr->rr_col[c];
2735 vdev_raidz_worst_error(raidz_row_t *rr)
2739 for (int c = 0; c < rr->rr_cols; c++) {
2740 error = zio_worst_error(error, rr->rr_col[c].rc_error);
2741 error = zio_worst_error(error, rr->rr_col[c].rc_shadow_error);
2748 vdev_raidz_io_done_verified(zio_t *zio, raidz_row_t *rr)
2757 for (int c = 0; c < rr->rr_cols; c++) {
2758 raidz_col_t *rc = &rr->rr_col[c];
2761 if (c < rr->rr_firstdatacol)
2768 } else if (c < rr->rr_firstdatacol && !rc->rc_tried) {
2785 rr->rr_firstdatacol - data_errors ||
2787 int n = raidz_parity_verify(zio, rr);
2796 for (int c = 0; c < rr->rr_cols; c++) {
2797 raidz_col_t *rc = &rr->rr_col[c];
2836 for (int c = 0; c < rr->rr_cols; c++) {
2837 raidz_col_t *rc = &rr->rr_col[c];
2868 raidz_row_t *rr = rm->rm_row[i];
2869 for (int c = 0; c < rr->rr_cols; c++) {
2870 raidz_col_t *rc = &rr->rr_col[c];
2938 raidz_row_t *rr = rm->rm_row[r];
2947 for (int c = 0; c < rr->rr_cols; c++) {
2948 raidz_col_t *rc = &rr->rr_col[c];
3010 vdev_raidz_reconstruct_row(rm, rr, my_tgts, t);
3020 raidz_row_t *rr = rm->rm_row[i];
3022 for (int c = 0; c < rr->rr_cols; c++) {
3023 raidz_col_t *rc = &rr->rr_col[c];
3036 c >= rr->rr_firstdatacol) {
3046 vdev_raidz_io_done_verified(zio, rr);
3126 raidz_row_t *rr = rm->rm_row[i];
3129 for (int c = 0; c < rr->rr_cols; c++) {
3130 if (rr->rr_col[c].rc_error)
3135 return (vdev_raidz_worst_error(rr));
3225 raidz_row_t *rr = rm->rm_row[row];
3226 vdev_raidz_reconstruct_row(rm, rr, t, nt);
3240 vdev_raidz_io_done_write_impl(zio_t *zio, raidz_row_t *rr)
3245 ASSERT3U(rr->rr_missingparity, <=, rr->rr_firstdatacol);
3246 ASSERT3U(rr->rr_missingdata, <=, rr->rr_cols - rr->rr_firstdatacol);
3249 for (int c = 0; c < rr->rr_cols; c++) {
3250 raidz_col_t *rc = &rr->rr_col[c];
3275 if (normal_errors > rr->rr_firstdatacol ||
3276 shadow_errors > rr->rr_firstdatacol) {
3278 vdev_raidz_worst_error(rr));
3284 raidz_row_t *rr)
3291 ASSERT3U(rr->rr_missingparity, <=, rr->rr_firstdatacol);
3292 ASSERT3U(rr->rr_missingdata, <=, rr->rr_cols - rr->rr_firstdatacol);
3294 for (int c = 0; c < rr->rr_cols; c++) {
3295 raidz_col_t *rc = &rr->rr_col[c];
3311 if (c < rr->rr_firstdatacol)
3317 } else if (c < rr->rr_firstdatacol && !rc->rc_tried) {
3328 total_errors <= rr->rr_firstdatacol - parity_untried) {
3337 ASSERT(parity_errors < rr->rr_firstdatacol);
3344 for (int c = rr->rr_firstdatacol; c < rr->rr_cols; c++) {
3345 raidz_col_t *rc = &rr->rr_col[c];
3352 ASSERT(rr->rr_firstdatacol >= n);
3354 vdev_raidz_reconstruct_row(rm, rr, tgts, n);
3362 vdev_raidz_read_all(zio_t *zio, raidz_row_t *rr)
3367 rr->rr_missingdata = 0;
3368 rr->rr_missingparity = 0;
3375 if (rr->rr_nempty != 0 && rr->rr_abd_empty == NULL)
3376 vdev_draid_map_alloc_empty(zio, rr);
3378 for (int c = 0; c < rr->rr_cols; c++) {
3379 raidz_col_t *rc = &rr->rr_col[c];
3405 raidz_row_t *rr = rm->rm_row[i];
3407 for (int c = 0; c < rr->rr_cols; c++) {
3408 raidz_col_t *rc = &rr->rr_col[c];
3444 raidz_row_t *rr = rm->rm_row[i];
3446 for (int c = 0; c < rr->rr_cols; c++) {
3447 raidz_col_t *rc = &rr->rr_col[c];
3456 if (c >= rr->rr_firstdatacol) {
3475 raidz_row_t *rr = rm->rm_row[i];
3477 rm, rr);
3485 raidz_row_t *rr = rm->rm_row[i];
3486 vdev_raidz_io_done_verified(zio, rr);