From 665454b73ef5897ac75657d78f132b5062cb8cfa Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 11 Jun 2024 21:39:19 +0000 Subject: [PATCH] remove zia --- include/sys/zia.h | 9 +- module/os/linux/zfs/vdev_disk.c | 55 ----- module/os/linux/zfs/vdev_file.c | 44 +--- module/zfs/spa.c | 17 +- module/zfs/vdev_raidz.c | 360 +++----------------------------- module/zfs/zia.c | 56 +++-- module/zfs/zia_cddl.c | 2 +- module/zfs/zio.c | 228 ++------------------ module/zfs/zio_checksum.c | 57 +---- 9 files changed, 111 insertions(+), 717 deletions(-) diff --git a/include/sys/zia.h b/include/sys/zia.h index a53e66ba3610..7f60f0d5cc0b 100644 --- a/include/sys/zia.h +++ b/include/sys/zia.h @@ -152,8 +152,10 @@ int zia_onload(void **handle, void *buf, size_t size); /* calls abd_iterate_func on the abd to copy abd data back and forth */ int zia_offload_abd(void *provider, abd_t *abd, - size_t size, size_t min_offload_size, boolean_t *local_offload); -int zia_onload_abd(abd_t *abd, size_t size, boolean_t keep_handle); + size_t size, size_t min_offload_size, + boolean_t *local_offload, boolean_t lock); +int zia_onload_abd(abd_t *abd, size_t size, + boolean_t keep_handle, boolean_t lock); /* move a handle into an abd */ void zia_move_into_abd(abd_t *dst, void **src); int zia_free_abd(abd_t *abd, boolean_t lock); @@ -162,7 +164,8 @@ int zia_free_abd(abd_t *abd, boolean_t lock); * if offloaded locally, just free the handle * if not, onload the data and free the handle */ -int zia_cleanup_abd(abd_t *abd, size_t size, boolean_t local_offload); +int zia_cleanup_abd(abd_t *abd, size_t size, + boolean_t local_offload, boolean_t lock); /* if the accelerator failed, restart the zio */ void zia_restart_before_vdev(zio_t *zio); diff --git a/module/os/linux/zfs/vdev_disk.c b/module/os/linux/zfs/vdev_disk.c index 443233b94342..7e4988055622 100644 --- a/module/os/linux/zfs/vdev_disk.c +++ b/module/os/linux/zfs/vdev_disk.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include #include @@ -302,7 +301,6 @@ vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize, reread_part = B_TRUE; } - zia_disk_close(v); vdev_blkdev_put(bdev, mode, zfs_vdev_holder); } @@ -421,11 +419,6 @@ vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize, *logical_ashift = highbit64(MAX(logical_block_size, SPA_MINBLOCKSIZE)) - 1; - zia_get_props(v->vdev_spa)->min_offload_size = 2 << *physical_ashift; - - /* open disk; ignore errors - will fall back to ZFS */ - zia_disk_open(v, v->vdev_path, vd->vd_bdev); - return (0); } @@ -438,7 +431,6 @@ vdev_disk_close(vdev_t *v) return; if (vd->vd_bdev != NULL) { - zia_disk_close(v); vdev_blkdev_put(vd->vd_bdev, spa_mode(v->vdev_spa), zfs_vdev_holder); } @@ -801,8 +793,6 @@ __vdev_disk_physio(struct block_device *bdev, zio_t *zio, return (error); } -EXPORT_SYMBOL(__vdev_disk_physio); - BIO_END_IO_PROTO(vdev_disk_io_flush_completion, bio, error) { zio_t *zio = bio->bi_private; @@ -845,8 +835,6 @@ vdev_disk_io_flush(struct block_device *bdev, zio_t *zio) return (0); } -EXPORT_SYMBOL(vdev_disk_io_flush); - static int vdev_disk_io_trim(zio_t *zio) { @@ -925,17 +913,6 @@ vdev_disk_io_start(zio_t *zio) break; } - error = zia_disk_flush(v, zio); - - /* - * have to return here in order to not dispatch - * this zio to multiple task queues - */ - if (error == 0) { - rw_exit(&vd->vd_lock); - return; - } - error = vdev_disk_io_flush(vd->vd_bdev, zio); if (error == 0) { rw_exit(&vd->vd_lock); @@ -975,37 +952,6 @@ vdev_disk_io_start(zio_t *zio) } zio->io_target_timestamp = zio_handle_io_delay(zio); - - error = EIO; - - if (rw == WRITE) { - boolean_t local_offload = B_FALSE; - zia_props_t *zia_props = zia_get_props(zio->io_spa); - if ((zia_props->disk_write == 1) && - (zio->io_can_offload == B_TRUE)) { - if (zia_offload_abd(zia_props->provider, zio->io_abd, - zio->io_size, zia_props->min_offload_size, - &local_offload) == ZIA_OK) { - error = zia_disk_write(v, zio, zio->io_size, - zio->io_offset, 0); - } - } - - if (error == 0) { - rw_exit(&vd->vd_lock); - return; - } - - error = zia_cleanup_abd(zio->io_abd, zio->io_size, - local_offload); - - if (error == ZIA_ACCELERATOR_DOWN) { - zia_disable_offloading(zio, B_TRUE); - rw_exit(&vd->vd_lock); - zio_interrupt(zio); - return; - } - } error = __vdev_disk_physio(vd->vd_bdev, zio, zio->io_size, zio->io_offset, rw, 0); rw_exit(&vd->vd_lock); @@ -1030,7 +976,6 @@ vdev_disk_io_done(zio_t *zio) vdev_disk_t *vd = v->vdev_tsd; if (!zfs_check_disk_status(vd->vd_bdev)) { - zia_disk_invalidate(v); invalidate_bdev(vd->vd_bdev); v->vdev_remove_wanted = B_TRUE; spa_async_request(zio->io_spa, SPA_ASYNC_REMOVE); diff --git a/module/os/linux/zfs/vdev_file.c b/module/os/linux/zfs/vdev_file.c index b50be29a118e..09e175cf9deb 100644 --- a/module/os/linux/zfs/vdev_file.c +++ b/module/os/linux/zfs/vdev_file.c @@ -36,7 +36,6 @@ #include #include #include -#include #ifdef _KERNEL #include #endif @@ -162,12 +161,6 @@ vdev_file_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize, } #endif - zia_get_props(vd->vdev_spa)->min_offload_size = 2 << *physical_ashift; - - /* try to open the file; ignore errors - will fall back to ZFS */ - zia_file_open(vd, vd->vdev_path, - vdev_file_open_mode(spa_mode(vd->vdev_spa)), 0); - skip_open: error = zfs_file_getattr(vf->vf_file, &zfa); @@ -191,8 +184,6 @@ vdev_file_close(vdev_t *vd) if (vd->vdev_reopening || vf == NULL) return; - zia_file_close(vd); - if (vf->vf_file != NULL) { (void) zfs_file_close(vf->vf_file); } @@ -212,47 +203,18 @@ vdev_file_io_strategy(void *arg) void *buf; loff_t off; ssize_t size; - int err = 0; + int err; off = zio->io_offset; size = zio->io_size; resid = 0; if (zio->io_type == ZIO_TYPE_READ) { - buf = abd_borrow_buf(zio->io_abd, size); + buf = abd_borrow_buf(zio->io_abd, zio->io_size); err = zfs_file_pread(vf->vf_file, buf, size, off, &resid); abd_return_buf_copy(zio->io_abd, buf, size); } else { - err = EIO; - - boolean_t local_offload = B_FALSE; - zia_props_t *zia_props = zia_get_props(zio->io_spa); - if ((zia_props->file_write == 1) && - (zio->io_can_offload == B_TRUE)) { - if (zia_offload_abd(zia_props->provider, - zio->io_abd, size, zia_props->min_offload_size, - &local_offload) == ZIA_OK) { - err = zia_file_write(vd, zio->io_abd, - size, off, &resid, &err); - } - } - - if (err == 0) { - zio->io_error = err; - if (resid != 0 && zio->io_error == 0) - zio->io_error = SET_ERROR(ENOSPC); - - zio_delay_interrupt(zio); - return; - } - - err = zia_cleanup_abd(zio->io_abd, size, local_offload); - if (err == ZIA_ACCELERATOR_DOWN) { - zia_disable_offloading(zio, B_TRUE); - zio_delay_interrupt(zio); - return; - } - buf = abd_borrow_buf_copy(zio->io_abd, size); + buf = abd_borrow_buf_copy(zio->io_abd, zio->io_size); err = zfs_file_pwrite(vf->vf_file, buf, size, off, &resid); abd_return_buf(zio->io_abd, buf, size); } diff --git a/module/zfs/spa.c b/module/zfs/spa.c index fca64131a85b..18ed867e484c 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -9075,7 +9075,8 @@ spa_sync_props(void *arg, dmu_tx_t *tx) * copied from zfs_ioc_pool_reopen */ spa_vdev_state_enter(spa, SCL_NONE); - vdev_reopen(spa->spa_root_vdev); + vdev_close(spa->spa_root_vdev); + (void) vdev_open(spa->spa_root_vdev); (void) spa_vdev_state_exit(spa, NULL, 0); spa_history_log_internal(spa, "set", tx, @@ -9165,12 +9166,26 @@ spa_sync_props(void *arg, dmu_tx_t *tx) case ZPOOL_PROP_ZIA_FILE_WRITE: zia_props->file_write = fnvpair_value_uint64(elem); + + /* reopen devices so that provider is used */ + spa_vdev_state_enter(spa, SCL_NONE); + vdev_close(spa->spa_root_vdev); + (void) vdev_open(spa->spa_root_vdev); + (void) spa_vdev_state_exit(spa, NULL, 0); + zia_prop_warn(zia_props->file_write, "File Write"); break; case ZPOOL_PROP_ZIA_DISK_WRITE: zia_props->disk_write = fnvpair_value_uint64(elem); + + /* reopen devices so that provider is used */ + spa_vdev_state_enter(spa, SCL_NONE); + vdev_close(spa->spa_root_vdev); + (void) vdev_open(spa->spa_root_vdev); + (void) spa_vdev_state_exit(spa, NULL, 0); + zia_prop_warn(zia_props->disk_write, "Disk Write"); break; diff --git a/module/zfs/vdev_raidz.c b/module/zfs/vdev_raidz.c index 5c00bfaa43d9..b0a655220882 100644 --- a/module/zfs/vdev_raidz.c +++ b/module/zfs/vdev_raidz.c @@ -36,7 +36,6 @@ #include #include #include -#include #ifdef ZFS_DEBUG #include /* For vdev_xlate() in vdev_raidz_io_verify() */ @@ -139,8 +138,6 @@ static void vdev_raidz_row_free(raidz_row_t *rr) { - zia_raidz_free(rr, B_FALSE); - for (int c = 0; c < rr->rr_cols; c++) { raidz_col_t *rc = &rr->rr_col[c]; @@ -362,7 +359,6 @@ vdev_raidz_map_alloc(zio_t *zio, uint64_t ashift, uint64_t dcols, rr->rr_offset = zio->io_offset; rr->rr_size = zio->io_size; #endif - rr->rr_zia_handle = NULL; asize = 0; @@ -525,8 +521,6 @@ vdev_raidz_generate_parity_p(raidz_row_t *rr) } } -EXPORT_SYMBOL(vdev_raidz_generate_parity_p); - void vdev_raidz_generate_parity_pq(raidz_row_t *rr) { @@ -569,8 +563,6 @@ vdev_raidz_generate_parity_pq(raidz_row_t *rr) } } -EXPORT_SYMBOL(vdev_raidz_generate_parity_pq); - void vdev_raidz_generate_parity_pqr(raidz_row_t *rr) { @@ -619,8 +611,6 @@ vdev_raidz_generate_parity_pqr(raidz_row_t *rr) } } -EXPORT_SYMBOL(vdev_raidz_generate_parity_pqr); - /* * Generate RAID parity in the first virtual columns according to the number of * parity columns available. @@ -1427,8 +1417,6 @@ vdev_raidz_reconstruct_general(raidz_row_t *rr, int *tgts, int ntgts) } } -EXPORT_SYMBOL(vdev_raidz_reconstruct_general); - static void vdev_raidz_reconstruct_row(raidz_map_t *rm, raidz_row_t *rr, const int *t, int nt) @@ -1646,24 +1634,7 @@ vdev_raidz_io_start_write(zio_t *zio, raidz_row_t *rr, uint64_t ashift) vdev_t *vd = zio->io_vd; raidz_map_t *rm = zio->io_vsd; - /* - * here instead of vdev_raidz_map_alloc or - * vdev_raidz_generate_parity_row to not have to - * store local_offload and be able to use zio - */ - boolean_t local_offload = B_FALSE; - if ((zia_raidz_alloc(zio, rr, B_FALSE, 0, &local_offload) != ZIA_OK) || - (zia_raidz_gen(rr) != ZIA_OK)) { - if (zia_raidz_gen_cleanup(zio, rr, - local_offload) == ZIA_ACCELERATOR_DOWN) { - zia_disable_offloading(zio, B_TRUE); - zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS >> 1; - return; - } - vdev_raidz_generate_parity_row(rm, rr); - } else { - zio->io_flags |= ZIO_FLAG_DONT_AGGREGATE; - } + vdev_raidz_generate_parity_row(rm, rr); for (int c = 0; c < rr->rr_scols; c++) { raidz_col_t *rc = &rr->rr_col[c]; @@ -1816,69 +1787,14 @@ raidz_checksum_verify(zio_t *zio) { zio_bad_cksum_t zbc = {0}; raidz_map_t *rm = zio->io_vsd; - /* - * if the zio entered this function offloaded, - * need to onload the parity columns on error - */ - const boolean_t entered_offloaded = zia_is_offloaded(zio->io_abd); int ret = zio_checksum_error(zio, &zbc); if (ret != 0 && zbc.zbc_injected != 0) rm->rm_ecksuminjected = 1; - /* - * zio_checksum_error does not get access to - * rm, so only the abd is freed on error - - * clean up rm here - */ - if (zia_is_offloaded(zio->io_abd) != B_TRUE) { - for (int i = 0; i < rm->rm_nrows; i++) { - raidz_row_t *rr = rm->rm_row[i]; - - /* - * force onload, since data was modified - * - * ignore return value - will always return ZIA_ERROR - */ - zia_raidz_rec_cleanup(zio, rr, B_TRUE, - entered_offloaded); - } - } - return (ret); } -static void -raidz_move_orig_parity(zio_t *zio, raidz_row_t *rr, abd_t **orig) -{ - (void) zio; - - for (uint64_t c = 0; c < rr->rr_firstdatacol; c++) { - raidz_col_t *rc = &rr->rr_col[c]; - if (!rc->rc_tried || rc->rc_error != 0) - continue; - - orig[c] = rc->rc_abd; - ASSERT3U(abd_get_size(rc->rc_abd), ==, rc->rc_size); - rc->rc_abd = abd_alloc_linear(rc->rc_size, B_FALSE); - zia_raidz_new_parity(zio, rr, c); - } -} - -static void -raidz_restore_orig_parity(raidz_row_t *rr, abd_t **orig) -{ - for (uint64_t c = 0; c < rr->rr_firstdatacol; c++) { - raidz_col_t *rc = &rr->rr_col[c]; - if (!rc->rc_tried || rc->rc_error != 0) - continue; - - abd_free(rc->rc_abd); - rc->rc_abd = orig[c]; - orig[c] = NULL; - } -} - /* * Generate the parity from the data columns. If we tried and were able to * read the parity without error, verify that the generated parity matches the @@ -1886,10 +1802,10 @@ raidz_restore_orig_parity(raidz_row_t *rr, abd_t **orig) * number of such failures. */ static int -raidz_parity_verify(zio_t *zio, raidz_row_t *rr, int *unexpected_errors) +raidz_parity_verify(zio_t *zio, raidz_row_t *rr) { - abd_t *orig[VDEV_RAIDZ_MAXPARITY] = { NULL }; - int c; + abd_t *orig[VDEV_RAIDZ_MAXPARITY]; + int c, ret = 0; raidz_map_t *rm = zio->io_vsd; raidz_col_t *rc; @@ -1898,45 +1814,31 @@ raidz_parity_verify(zio_t *zio, raidz_row_t *rr, int *unexpected_errors) (BP_IS_GANG(bp) ? ZIO_CHECKSUM_GANG_HEADER : BP_GET_CHECKSUM(bp))); if (checksum == ZIO_CHECKSUM_NOPARITY) - return (0); + return (ret); - raidz_move_orig_parity(zio, rr, orig); + for (c = 0; c < rr->rr_firstdatacol; c++) { + rc = &rr->rr_col[c]; + if (!rc->rc_tried || rc->rc_error != 0) + continue; + + orig[c] = rc->rc_abd; + ASSERT3U(abd_get_size(rc->rc_abd), ==, rc->rc_size); + rc->rc_abd = abd_alloc_linear(rc->rc_size, B_FALSE); + } /* * Verify any empty sectors are zero filled to ensure the parity * is calculated correctly even if these non-data sectors are damaged. */ if (rr->rr_nempty && rr->rr_abd_empty != NULL) - *unexpected_errors += vdev_draid_map_verify_empty(zio, rr); + ret += vdev_draid_map_verify_empty(zio, rr); /* * Regenerates parity even for !tried||rc_error!=0 columns. This * isn't harmful but it does have the side effect of fixing stuff * we didn't realize was necessary (i.e. even if we return 0). */ - if (zia_raidz_gen(rr) != ZIA_OK) { - /* - * restore original parity columns so - * that the reconstructed parity can - * be brought back with the data columns - */ - raidz_restore_orig_parity(rr, orig); - - /* return reconstructed columns to memory */ - const int ret = zia_raidz_rec_cleanup(zio, rr, - B_FALSE, B_TRUE); - - if (ret == ZIA_ACCELERATOR_DOWN) { - return (ret); - } - - /* - * continue to software, so redo the - * original moving of parity columns - */ - raidz_move_orig_parity(zio, rr, orig); - vdev_raidz_generate_parity_row(rm, rr); - } + vdev_raidz_generate_parity_row(rm, rr); for (c = 0; c < rr->rr_firstdatacol; c++) { rc = &rr->rr_col[c]; @@ -1944,77 +1846,15 @@ raidz_parity_verify(zio_t *zio, raidz_row_t *rr, int *unexpected_errors) if (!rc->rc_tried || rc->rc_error != 0) continue; - int cmp = 0; - if (zia_raidz_cmp(orig[c], rc->rc_abd, &cmp) != ZIA_OK) { - if (zia_is_offloaded(zio->io_abd) || - rr->rr_zia_handle) { - /* - * should only need to onload orig[c] and - * rc but onloading everything to not create - * inconsistent rr state - */ - int ret = zia_raidz_rec_cleanup(zio, rr, - B_FALSE, B_TRUE); - - for (uint64_t i = 0; i < rr->rr_firstdatacol; - i++) { - if (orig[i]) { - ret = zia_worst_error(ret, - zia_onload_abd(orig[i], - orig[i]->abd_size, - B_FALSE)); - } - } - - if (ret == ZIA_ACCELERATOR_DOWN) { - /* - * get original parity columns back to - * get the original in-memory data - */ - raidz_restore_orig_parity(rr, orig); - return (ret); - } - } - cmp = abd_cmp(orig[c], rc->rc_abd); - } - if (cmp != 0) { - if (zia_is_offloaded(zio->io_abd) || - rr->rr_zia_handle) { - /* - * should only need to onload orig[c] and - * rc but onloading everything to not create - * inconsistent rr state - */ - int ret = zia_raidz_rec_cleanup(zio, rr, - B_FALSE, B_TRUE); - - for (uint64_t i = 0; i < rr->rr_firstdatacol; - i++) { - if (orig[i]) { - ret = zia_worst_error(ret, - zia_onload_abd(orig[i], - orig[i]->abd_size, - B_FALSE)); - } - } - - if (ret == ZIA_ACCELERATOR_DOWN) { - /* - * get original parity columns back to - * get the original in-memory data - */ - raidz_restore_orig_parity(rr, orig); - return (ret); - } - } + if (abd_cmp(orig[c], rc->rc_abd) != 0) { vdev_raidz_checksum_error(zio, rc, orig[c]); rc->rc_error = SET_ERROR(ECKSUM); - (*unexpected_errors)++; + ret++; } abd_free(orig[c]); } - return (0); + return (ret); } static int @@ -2028,7 +1868,7 @@ vdev_raidz_worst_error(raidz_row_t *rr) return (error); } -static int +static void vdev_raidz_io_done_verified(zio_t *zio, raidz_row_t *rr) { int unexpected_errors = 0; @@ -2068,11 +1908,8 @@ vdev_raidz_io_done_verified(zio_t *zio, raidz_row_t *rr) if (parity_errors + parity_untried < rr->rr_firstdatacol - data_errors || (zio->io_flags & ZIO_FLAG_RESILVER)) { - int ret = - raidz_parity_verify(zio, rr, &unexpected_errors); - if (ret != 0) { - return (ret); - } + int n = raidz_parity_verify(zio, rr); + unexpected_errors += n; } if (zio->io_error == 0 && spa_writeable(zio->io_spa) && @@ -2101,7 +1938,6 @@ vdev_raidz_io_done_verified(zio_t *zio, raidz_row_t *rr) ZIO_FLAG_SELF_HEAL : 0), NULL, NULL)); } } - return (0); } static void @@ -2170,42 +2006,15 @@ raidz_reconstruct(zio_t *zio, int *ltgts, int ntgts, int nparity) } if (dead > nparity) { /* reconstruction not possible */ - /* drop offloaded data */ - for (int i = 0; i < rm->rm_nrows; i++) { - raidz_row_t *rr = rm->rm_row[i]; - zia_raidz_rec_cleanup(zio, rr, B_TRUE, B_FALSE); - /* no data movement, so errors don't matter */ - } raidz_restore_orig_data(rm); return (EINVAL); } - - if (dead_data > 0) { - /* - * here instead of vdev_raidz_reconstruct_row - * to be able to use zio - */ - if (zia_raidz_rec(rr, my_tgts, t) != ZIA_OK) { - int ret = ZIA_OK; - for (int i = 0; i < rm->rm_nrows; i++) { - raidz_row_t *rr = rm->rm_row[i]; - ret = zia_worst_error(ret, - zia_raidz_rec_cleanup(zio, rr, - B_FALSE, B_TRUE)); - } - - if ((ret != ZIA_OK) && - (ret != ZIA_ACCELERATOR_DOWN)) { - vdev_raidz_reconstruct_row(rm, rr, - my_tgts, t); - } - } - } + if (dead_data > 0) + vdev_raidz_reconstruct_row(rm, rr, my_tgts, t); } /* Check for success */ if (raidz_checksum_verify(zio) == 0) { - int ret = 0; /* Reconstruction succeeded - report errors */ for (int i = 0; i < rm->rm_nrows; i++) { @@ -2235,27 +2044,15 @@ raidz_reconstruct(zio_t *zio, int *ltgts, int ntgts, int nparity) } } - const int rc = - vdev_raidz_io_done_verified(zio, rr); - ret = zia_worst_error(ret, rc); + vdev_raidz_io_done_verified(zio, rr); } zio_checksum_verified(zio); - if (ret != ZIA_ACCELERATOR_DOWN) { - ret = 0; - } - - return (ret); + return (0); } /* Reconstruction failed - restore original data */ - /* drop offloaded data */ - for (int i = 0; i < rm->rm_nrows; i++) { - raidz_row_t *rr = rm->rm_row[i]; - zia_raidz_rec_cleanup(zio, rr, B_TRUE, B_FALSE); - /* no data movement, so errors don't matter */ - } raidz_restore_orig_data(rm); return (ECKSUM); } @@ -2347,9 +2144,6 @@ vdev_raidz_combrec(zio_t *zio) for (;;) { int err = raidz_reconstruct(zio, ltgts, num_failures, nparity); - if (err == ZIA_ACCELERATOR_DOWN) { - return (err); - } if (err == EINVAL) { /* * Reconstruction not possible with this # @@ -2522,18 +2316,6 @@ vdev_raidz_io_done_reconstruct_known_missing(zio_t *zio, raidz_map_t *rm, ASSERT(rr->rr_firstdatacol >= n); - if (zia_raidz_rec(rr, tgts, n) == ZIA_OK) { - return; - } - - /* - * drop handles instead of onloading - * - * return value doesn't matter because - * the data hasn't changed yet - */ - zia_raidz_rec_cleanup(zio, rr, - B_TRUE, B_FALSE); vdev_raidz_reconstruct_row(rm, rr, tgts, n); } } @@ -2618,88 +2400,16 @@ vdev_raidz_io_done(zio_t *zio) vdev_raidz_io_done_write_impl(zio, rm->rm_row[i]); } } else { - /* the raidz rows should never enter here already offloaded */ - for (int i = 0; i < rm->rm_nrows; i++) { - raidz_row_t *rr = rm->rm_row[i]; - ASSERT(rr->rr_zia_handle == NULL); - } - - /* offload once at beginning */ - blkptr_t *bp = zio->io_bp; - if (bp && !BP_IS_METADATA(bp)) { - uint_t checksum = (BP_IS_GANG(bp) ? - ZIO_CHECKSUM_GANG_HEADER : BP_GET_CHECKSUM(bp)); - zio_checksum_info_t *ci = &zio_checksum_table[checksum]; - if (!(ci->ci_flags & ZCHECKSUM_FLAG_EMBEDDED)) { - for (int i = 0; i < rm->rm_nrows; i++) { - raidz_row_t *rr = rm->rm_row[i]; - /* - * Allow unchecked failure since failure - * to offload means the software path - * will be taken. Whether or not the - * provider/offloader is valid - * becomes irrelevant. - */ - zia_raidz_alloc(zio, rr, - B_TRUE, checksum, NULL); - } - } - } - for (int i = 0; i < rm->rm_nrows; i++) { raidz_row_t *rr = rm->rm_row[i]; vdev_raidz_io_done_reconstruct_known_missing(zio, rm, rr); - /* - * Restarting here is unnecessary. If the offloader - * failed, the offloaded data is still in sync with - * the in-memory data, and falling back reconstructed - * using the correct data. - */ - } - - int ret = raidz_checksum_verify(zio); - - /* ZIA_ACCELERATOR_DOWN is a completely orthogonal error */ - if (ret == ZIA_ACCELERATOR_DOWN) { - for (int i = 0; i < rm->rm_nrows; i++) { - raidz_row_t *rr = rm->rm_row[i]; - zia_raidz_rec_cleanup(zio, rr, B_TRUE, B_FALSE); - } - - zio->io_can_offload = B_FALSE; - zio_vdev_io_redone(zio); - return; } - if (ret == 0) { + if (raidz_checksum_verify(zio) == 0) { for (int i = 0; i < rm->rm_nrows; i++) { raidz_row_t *rr = rm->rm_row[i]; - ret = - vdev_raidz_io_done_verified(zio, rr); - if (ret == ZIA_ACCELERATOR_DOWN) { - for (int j = 0; j < rm->rm_nrows; j++) { - rr = rm->rm_row[j]; - - /* - * vdev_raidz_io_done_verified - * will have already attempted - * to load reconstructed data - * back into memory, so this - * line should just drop any - * remaining handles - * - * not sure why onload_parity - * has to be set to B_TRUE - */ - zia_raidz_rec_cleanup(zio, rr, - B_TRUE, B_TRUE); - } - - zio->io_can_offload = B_FALSE; - zio_vdev_io_redone(zio); - return; - } + vdev_raidz_io_done_verified(zio, rr); } zio_checksum_verified(zio); } else { @@ -2726,12 +2436,6 @@ vdev_raidz_io_done(zio_t *zio) rm->rm_row[i]); } if (nread != 0) { - /* drop handles */ - for (int i = 0; i < rm->rm_nrows; i++) { - raidz_row_t *rr = rm->rm_row[i]; - zia_raidz_rec_cleanup(zio, rr, - B_TRUE, B_FALSE); - } /* * Normally our stage is VDEV_IO_DONE, but if * we've already called redone(), it will have @@ -2744,14 +2448,6 @@ vdev_raidz_io_done(zio_t *zio) } zio->io_error = vdev_raidz_combrec(zio); - - if (zio->io_error == ZIA_ACCELERATOR_DOWN) { - zio->io_error = 0; - zio->io_can_offload = B_FALSE; - zio_vdev_io_redone(zio); - return; - } - if (zio->io_error == ECKSUM && !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) { vdev_raidz_io_done_unrecoverable(zio); diff --git a/module/zfs/zia.c b/module/zfs/zia.c index f3eb3575745a..769b8bf85868 100644 --- a/module/zfs/zia.c +++ b/module/zfs/zia.c @@ -676,7 +676,8 @@ zia_offload_abd_offset(void *provider, abd_t *abd, int zia_offload_abd(void *provider, abd_t *abd, - size_t size, size_t min_offload_size, boolean_t *local_offload) + size_t size, size_t min_offload_size, + boolean_t *local_offload, boolean_t lock) { if (!dpusm) { return (ZIA_FALLBACK); @@ -686,8 +687,18 @@ zia_offload_abd(void *provider, abd_t *abd, return (ZIA_ERROR); } - return (zia_offload_abd_offset(provider, - abd, 0, size, min_offload_size, local_offload)); + if (lock) { + mutex_enter(&abd->abd_mtx); + } + + const int rc = zia_offload_abd_offset(provider, + abd, 0, size, min_offload_size, local_offload); + + if (lock) { + mutex_exit(&abd->abd_mtx); + } + + return (rc); } #ifdef ZIA @@ -768,7 +779,7 @@ zia_onload_abd_offset(abd_t *abd, size_t offset, } int -zia_onload_abd(abd_t *abd, size_t size, boolean_t keep_handle) +zia_onload_abd(abd_t *abd, size_t size, boolean_t keep_handle, boolean_t lock) { if (abd_is_gang(abd)) { /* @@ -792,7 +803,17 @@ zia_onload_abd(abd_t *abd, size_t size, boolean_t keep_handle) ASSERT(size <= original_size); } - return (zia_onload_abd_offset(abd, 0, size, keep_handle)); + if (lock == B_TRUE) { + mutex_enter(&abd->abd_mtx); + } + + const int err = zia_onload_abd_offset(abd, 0, size, keep_handle); + + if (lock == B_TRUE) { + mutex_exit(&abd->abd_mtx); + } + + return (err); } void @@ -822,7 +843,7 @@ zia_free_abd(abd_t *abd, boolean_t lock) * if not, onload the data and free the handle */ int -zia_cleanup_abd(abd_t *abd, size_t size, boolean_t local_offload) +zia_cleanup_abd(abd_t *abd, size_t size, boolean_t local_offload, boolean_t lock) { if (!dpusm) { return (ZIA_FALLBACK); @@ -836,10 +857,10 @@ zia_cleanup_abd(abd_t *abd, size_t size, boolean_t local_offload) if (local_offload == B_TRUE) { /* in-memory copy is still valid */ /* lock just in case mirrors clean up at the same time */ - ret = zia_free_abd(abd, B_FALSE); + ret = zia_free_abd(abd, B_TRUE); } else { /* have to copy data into memory */ - ret = zia_onload_abd(abd, size, B_FALSE); + ret = zia_onload_abd(abd, size, B_FALSE, lock); } return (ret); @@ -939,7 +960,7 @@ zia_decompress(zia_props_t *props, enum zio_compress c, } int ret = zia_offload_abd(props->provider, src, - s_len, props->min_offload_size, NULL); + s_len, props->min_offload_size, NULL, B_FALSE); if (ret != ZIA_OK) { return (ret); } @@ -999,7 +1020,7 @@ zia_checksum_compute(void *provider, zio_cksum_t *dst, enum zio_checksum alg, if (zia_offload_abd(provider, zio->io_abd, size, zia_get_props(zio->io_spa)->min_offload_size, - local_offload) != ZIA_OK) { + local_offload, B_FALSE) != ZIA_OK) { return (ZIA_ERROR); } } else { @@ -1107,7 +1128,7 @@ zia_raidz_cleanup(zio_t *zio, raidz_row_t *rr, mutex_enter(&zio->io_abd->abd_mtx); const int ret = zia_worst_error( zia_raidz_free(rr, onload_parity), - zia_cleanup_abd(zio->io_abd, zio->io_size, local_offload)); + zia_cleanup_abd(zio->io_abd, zio->io_size, local_offload, B_FALSE)); mutex_exit(&zio->io_abd->abd_mtx); return (ret); @@ -1192,7 +1213,7 @@ zia_raidz_alloc(zio_t *zio, raidz_row_t *rr, boolean_t rec, */ mutex_enter(&zio->io_abd->abd_mtx); const int ret = zia_offload_abd(provider, zio->io_abd, - zio->io_size, props->min_offload_size, local_offload); + zio->io_size, props->min_offload_size, local_offload, B_TRUE); mutex_exit(&zio->io_abd->abd_mtx); if (ret != ZIA_OK) { return (ret); @@ -1226,7 +1247,7 @@ zia_raidz_alloc(zio_t *zio, raidz_row_t *rr, boolean_t rec, */ zia_offload_abd(provider, rc->rc_abd, rc->rc_abd->abd_size, props->min_offload_size, - NULL); + NULL, B_FALSE); handle = ABD_HANDLE(rc->rc_abd); } else { /* generating - create new columns */ @@ -1331,7 +1352,7 @@ zia_raidz_free(raidz_row_t *rr, boolean_t onload_parity) raidz_col_t *rc = &rr->rr_col[c]; ret = zia_worst_error(ret, zia_onload_abd(rc->rc_abd, - rc->rc_size, B_FALSE)); + rc->rc_size, B_FALSE, B_FALSE)); } } @@ -1498,8 +1519,13 @@ zia_file_open(vdev_t *vdev, const char *path, return (ZIA_ERROR); } + zia_props_t *zia_props = zia_get_props(vdev->vdev_spa); + if (zia_props->file_write != 1) { + return (ZIA_FALLBACK); + } + #ifdef ZIA - void *provider = zia_get_props(vdev->vdev_spa)->provider; + void *provider = zia_props->provider; if (!dpusm || !provider) { return (ZIA_FALLBACK); } diff --git a/module/zfs/zia_cddl.c b/module/zfs/zia_cddl.c index 5a9041585340..557f240050aa 100644 --- a/module/zfs/zia_cddl.c +++ b/module/zfs/zia_cddl.c @@ -69,7 +69,7 @@ zia_compress_impl(const dpusm_uf_t *dpusm, zia_props_t *props, } ret = zia_offload_abd(props->provider, src, s_len, - props->min_offload_size, local_offload); + props->min_offload_size, local_offload, B_FALSE); if (ret != ZIA_OK) { return (ret); } diff --git a/module/zfs/zio.c b/module/zfs/zio.c index 6907682d98e8..3eb472a9fd2a 100644 --- a/module/zfs/zio.c +++ b/module/zfs/zio.c @@ -51,7 +51,6 @@ #include #include #include -#include #include /* @@ -395,12 +394,12 @@ zio_push_transform(zio_t *zio, abd_t *data, uint64_t size, uint64_t bufsize, zio->io_size = size; } -zio_transform_t * -zio_pop_transform(zio_t *zio) +void +zio_pop_transforms(zio_t *zio) { - zio_transform_t *zt = zio->io_transform_stack; + zio_transform_t *zt; - if (zt != NULL) { + while ((zt = zio->io_transform_stack) != NULL) { if (zt->zt_transform != NULL) zt->zt_transform(zio, zt->zt_orig_abd, zt->zt_orig_size); @@ -414,15 +413,6 @@ zio_pop_transform(zio_t *zio) kmem_free(zt, sizeof (zio_transform_t)); } - - return (zt); -} - -void -zio_pop_transforms(zio_t *zio) -{ - while (zio_pop_transform(zio)) { - } } /* @@ -443,42 +433,11 @@ static void zio_decompress(zio_t *zio, abd_t *data, uint64_t size) { if (zio->io_error == 0) { - int ret = ZIA_FALLBACK; - zia_props_t *zia_props = zia_get_props(zio->io_spa); - if ((zia_props->decompress == 1) && - (zio->io_can_offload == B_TRUE)) { - ret = zia_decompress(zia_props, - BP_GET_COMPRESS(zio->io_bp), - zio->io_abd, zio->io_size, - data, size, - &zio->io_prop.zp_complevel); - } - - if (ret == ZIA_OK) { - ASSERT(zia_is_offloaded(zio->io_abd) == B_TRUE); - /* - * bring data back into memory since there - * are no subsequent offloaded stages - */ - ret = zia_onload_abd(data, size, B_FALSE); - } - - ASSERT(zia_is_offloaded(data) != B_TRUE); - /* let abd_free clean up zio->io_abd */ - - if (ret == ZIA_OK) { - ret = 0; - } else { - if (ret == ZIA_ACCELERATOR_DOWN) { - zia_disable_offloading(zio, B_FALSE); - } - - void *tmp = abd_borrow_buf(data, size); - ret = zio_decompress_data(BP_GET_COMPRESS(zio->io_bp), - zio->io_abd, tmp, zio->io_size, size, - &zio->io_prop.zp_complevel); - abd_return_buf_copy(data, tmp, size); - } + void *tmp = abd_borrow_buf(data, size); + int ret = zio_decompress_data(BP_GET_COMPRESS(zio->io_bp), + zio->io_abd, tmp, zio->io_size, size, + &zio->io_prop.zp_complevel); + abd_return_buf_copy(data, tmp, size); if (zio_injection_enabled && ret == 0) ret = zio_handle_fault_injection(zio, EINVAL); @@ -789,11 +748,6 @@ zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait, if (zio->io_error && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) *errorp = zio_worst_error(*errorp, zio->io_error); pio->io_reexecute |= zio->io_reexecute; - if ((zio->io_flags & ZIO_FLAG_ZIA_REEXECUTE) && - (zio->io_can_offload != B_TRUE)) { - pio->io_flags |= ZIO_FLAG_ZIA_REEXECUTE; - pio->io_can_offload = B_FALSE; - } ASSERT3U(*countp, >, 0); (*countp)--; @@ -844,10 +798,6 @@ zio_inherit_child_errors(zio_t *zio, enum zio_child c) { if (zio->io_child_error[c] != 0 && zio->io_error == 0) zio->io_error = zio->io_child_error[c]; - - if (zio->io_flags & ZIO_FLAG_ZIA_REEXECUTE) { - zio->io_can_offload = B_FALSE; - } } int @@ -968,13 +918,7 @@ zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, if (zb != NULL) zio->io_bookmark = *zb; - zio->io_can_offload = zia_get_props(spa)->can_offload; - if (pio != NULL) { - if ((pio->io_flags & ZIO_FLAG_ZIA_REEXECUTE) || - (pio->io_can_offload != B_TRUE)) { - zio->io_can_offload = B_FALSE; - } zio->io_metaslab_class = pio->io_metaslab_class; if (zio->io_logical == NULL) zio->io_logical = pio->io_logical; @@ -983,13 +927,6 @@ zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, zio_add_child_first(pio, zio); } - /* turn off encryption and dedup if Z.I.A. is used */ - if (zia_is_used(zio) == B_TRUE) { - zio->io_prop.zp_dedup = B_FALSE; - zio->io_prop.zp_dedup_verify = B_FALSE; - zio->io_prop.zp_encrypt = B_FALSE; - } - taskq_init_ent(&zio->io_tqent); return (zio); @@ -1874,85 +1811,18 @@ zio_write_compress(zio_t *zio) if (compress != ZIO_COMPRESS_OFF && !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) { void *cbuf = NULL; - int zia_rc = ZIA_FALLBACK; - void *cbuf_handle = NULL; /* only valid if zia_rc == ZIA_OK */ - zia_props_t *zia_props = zia_get_props(spa); - boolean_t local_offload = B_FALSE; - if ((zia_props->compress == 1) && - (zio->io_can_offload == B_TRUE)) { - zia_rc = zia_compress(zia_props, compress, - zio->io_abd, lsize, &cbuf_handle, &psize, - zp->zp_complevel, &local_offload); - } - - if (zia_rc != ZIA_OK) { - ASSERT(cbuf_handle == NULL); - zia_rc = zia_cleanup_abd(zio->io_abd, - lsize, local_offload); - - /* - * if data has to be brought back for cpu compression, - * but could not, restart the pipeline for this zio - * (not necessary in this case, but still doing it here - * in case a previous stage is offloaded) - */ - if (zia_rc == ZIA_ACCELERATOR_DOWN) { - zia_restart_before_vdev(zio); - return (zio); - } - psize = zio_compress_data(compress, zio->io_abd, &cbuf, - lsize, zp->zp_complevel); - } - + psize = zio_compress_data(compress, zio->io_abd, &cbuf, lsize, + zp->zp_complevel); if (psize == 0) { compress = ZIO_COMPRESS_OFF; - ASSERT0(cbuf_handle); } else if (psize >= lsize) { compress = ZIO_COMPRESS_OFF; if (cbuf != NULL) zio_buf_free(cbuf, lsize); - /* - * no need for offloaded - * compressed buffer any more - * - * catch accelerator failures elsewhere - */ - zia_free(&cbuf_handle); - - /* source abd is still offloaded */ } else if (!zp->zp_dedup && !zp->zp_encrypt && psize <= BPE_PAYLOAD_SIZE && zp->zp_level == 0 && !DMU_OT_HAS_FILL(zp->zp_type) && spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA)) { - - /* - * compressed enough, but not handling embedded - * data, so move compressed data back into memory - */ - if (cbuf_handle) { - cbuf = zio_buf_alloc(lsize); - } - - if (zia_onload(&cbuf_handle, cbuf, - psize) == ZIA_ACCELERATOR_DOWN) { - zia_free_abd(zio->io_abd, B_FALSE); - zio_buf_free(cbuf, lsize); - zia_restart_before_vdev(zio); - return (zio); - } - - /* - * remove offloaded source abd - * - * in-memory copy should still be valid, but calling - * zia_cleanup_abd just in case - */ - if (zia_cleanup_abd(zio->io_abd, lsize, - local_offload) == ZIA_ACCELERATOR_DOWN) { - zio_buf_free(cbuf, lsize); - zia_restart_before_vdev(zio); - return (zio); - } encode_embedded_bp_compressed(bp, cbuf, compress, lsize, psize); BPE_SET_ETYPE(bp, BP_EMBEDDED_TYPE_DATA); @@ -1977,68 +1847,15 @@ zio_write_compress(zio_t *zio) psize); if (rounded >= lsize) { compress = ZIO_COMPRESS_OFF; - if (cbuf_handle) { - /* - * catch accelerator - * down elsewhere - */ - zia_free(&cbuf_handle); - } else { - zio_buf_free(cbuf, lsize); - } + zio_buf_free(cbuf, lsize); psize = lsize; } else { - /* abd_get_from_buf must not get a NULL */ - if (cbuf_handle) { - cbuf = zio_buf_alloc(lsize); - } abd_t *cdata = abd_get_from_buf(cbuf, lsize); abd_take_ownership_of_buf(cdata, B_TRUE); - if (cbuf_handle) { - /* - * zio->io_abd offload no longer needed - * but don't free here - let abd_free - * handle it - */ - - /* - * compressed enough, so associate the - * compressed buffer with the abd - */ - zia_move_into_abd(cdata, &cbuf_handle); - zia_rc = zia_zero_fill(cdata, psize, - rounded - psize); - if (zia_rc != ZIA_OK) { - /* - * if setting cdata's handle - * fails, onload the compressed - * buffer (automatically placing - * it into cdata) and continue - * using zfs - * - * if cbuf is not offloaded, - * nothing happens - */ - zia_rc = zia_onload( - &cbuf_handle, cbuf, lsize); - } - - if (zia_rc == ZIA_ACCELERATOR_DOWN) { - zia_free(&cbuf_handle); - zia_free_abd(zio->io_abd, - B_FALSE); - zia_restart_before_vdev(zio); - return (zio); - } - } abd_zero_off(cdata, psize, rounded - psize); psize = rounded; zio_push_transform(zio, cdata, psize, lsize, NULL); - if (zia_is_offloaded(zio->io_abd)) { - zio->io_flags |= - ZIO_FLAG_DONT_AGGREGATE; - } } } @@ -4107,13 +3924,6 @@ zio_vdev_io_start(zio_t *zio) if (zio->io_type == ZIO_TYPE_WRITE) { abd_copy(abuf, zio->io_abd, zio->io_size); abd_zero_off(abuf, zio->io_size, asize - zio->io_size); - /* - * The Z.I.A. handles of the abds that come here - * were not modified and do not get associated with - * abuf during the transform. Instead of dropping - * the handle and delaying here, let abd_free clean - * it up later. - */ } zio_push_transform(zio, abuf, asize, asize, zio_subblock); } @@ -4304,8 +4114,6 @@ zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr) { void *abd = abd_alloc_sametype(zio->io_abd, zio->io_size); - zia_onload_abd(zio->io_abd, zio->io_size, B_FALSE); - abd_copy(abd, zio->io_abd, zio->io_size); zcr->zcr_cbinfo = zio->io_size; @@ -4340,9 +4148,7 @@ zio_vdev_io_assess(zio_t *zio) * On retry, we cut in line in the issue queue, since we don't want * compression/checksumming/etc. work to prevent our (cheap) IO reissue. */ - if (zio->io_error && - !(zio->io_flags & ZIO_FLAG_ZIA_REEXECUTE) && - vd == NULL && + if (zio->io_error && vd == NULL && !(zio->io_flags & (ZIO_FLAG_DONT_RETRY | ZIO_FLAG_IO_RETRY))) { ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_QUEUE)); /* not a leaf */ ASSERT(!(zio->io_flags & ZIO_FLAG_IO_BYPASS)); /* not a leaf */ @@ -4967,8 +4773,6 @@ zio_done(zio_t *zio) } if (zio->io_error) { - ASSERT(!(zio->io_flags & ZIO_FLAG_ZIA_REEXECUTE)); - /* * If this I/O is attached to a particular vdev, * generate an error message describing the I/O failure @@ -5003,10 +4807,7 @@ zio_done(zio_t *zio) } } - if ((zio->io_error || - (zio->io_flags & ZIO_FLAG_ZIA_REEXECUTE) || - 0) && - zio == zio->io_logical) { + if (zio->io_error && zio == zio->io_logical) { /* * Determine whether zio should be reexecuted. This will * propagate all the way to the root via zio_notify_parent(). @@ -5378,7 +5179,6 @@ EXPORT_SYMBOL(zio_buf_alloc); EXPORT_SYMBOL(zio_data_buf_alloc); EXPORT_SYMBOL(zio_buf_free); EXPORT_SYMBOL(zio_data_buf_free); -EXPORT_SYMBOL(zio_push_transform); ZFS_MODULE_PARAM(zfs_zio, zio_, slow_io_ms, INT, ZMOD_RW, "Max I/O completion time (milliseconds) before marking it as slow"); diff --git a/module/zfs/zio_checksum.c b/module/zfs/zio_checksum.c index 44b9d7445a75..9de515e8767a 100644 --- a/module/zfs/zio_checksum.c +++ b/module/zfs/zio_checksum.c @@ -31,7 +31,6 @@ #include #include #include -#include #include /* @@ -358,13 +357,6 @@ zio_checksum_compute(zio_t *zio, enum zio_checksum checksum, zio_eck_t eck; size_t eck_offset; - /* not handling embedded checksums, so bring back data */ - const int zia_rc = zia_cleanup_abd(abd, size, B_FALSE); - if (zia_rc == ZIA_ACCELERATOR_DOWN) { - zia_restart_before_vdev(zio); - return; - } - memset(&saved, 0, sizeof (zio_cksum_t)); if (checksum == ZIO_CHECKSUM_ZILOG2) { @@ -408,30 +400,8 @@ zio_checksum_compute(zio_t *zio, enum zio_checksum checksum, sizeof (zio_cksum_t)); } else { saved = bp->blk_cksum; - - int zia_rc = ZIA_FALLBACK; - - /* only offload non-embedded checksums */ - boolean_t local_offload = B_FALSE; - zia_props_t *zia_props = zia_get_props(spa); - if ((zia_props->checksum == 1) && - (zio->io_can_offload == B_TRUE)) { - zia_rc = zia_checksum_compute(zia_props->provider, - &cksum, checksum, zio, size, &local_offload); - } - - /* fall back to ZFS implementation */ - if (zia_rc != ZIA_OK) { - zia_rc = zia_cleanup_abd(abd, size, local_offload); - if (zia_rc == ZIA_ACCELERATOR_DOWN) { - zia_restart_before_vdev(zio); - return; - } ci->ci_func[0](abd, size, spa->spa_cksum_tmpls[checksum], &cksum); - } else { - zio->io_flags |= ZIO_FLAG_DONT_AGGREGATE; - } if (BP_USES_CRYPT(bp) && BP_GET_TYPE(bp) != DMU_OT_OBJSET) zio_checksum_handle_crypt(&cksum, &saved, insecure); bp->blk_cksum = cksum; @@ -460,12 +430,6 @@ zio_checksum_error_impl(spa_t *spa, const blkptr_t *bp, zio_cksum_t verifier; size_t eck_offset; - /* not handling embedded checksums, so bring back data */ - const int zia_rc = zia_cleanup_abd(abd, size, B_FALSE); - if (zia_rc == ZIA_ACCELERATOR_DOWN) { - return (zia_rc); - } - if (checksum == ZIO_CHECKSUM_ZILOG2) { zil_chain_t zilc; uint64_t nused; @@ -526,25 +490,8 @@ zio_checksum_error_impl(spa_t *spa, const blkptr_t *bp, } else { byteswap = BP_SHOULD_BYTESWAP(bp); expected_cksum = bp->blk_cksum; - - zia_props_t *zia_props = zia_get_props(spa); - int error = ZIA_FALLBACK; - if ((zia_props->can_offload == B_TRUE) && - (zia_props->checksum == 1)) { - error = zia_checksum_error(checksum, abd, size, - byteswap, &actual_cksum); - } - - /* fall back to ZFS implementation */ - if ((error != ZIA_OK) && (error != ECKSUM)) { - /* data was modified by reconstruction */ - error = zia_onload_abd(abd, size, B_FALSE); - if (error == ZIA_ACCELERATOR_DOWN) { - return (error); - } - ci->ci_func[byteswap](abd, size, - spa->spa_cksum_tmpls[checksum], &actual_cksum); - } + ci->ci_func[byteswap](abd, size, + spa->spa_cksum_tmpls[checksum], &actual_cksum); } /*