diff --git a/include/sys/zia.h b/include/sys/zia.h index 7f60f0d5cc0b..19a9ec095164 100644 --- a/include/sys/zia.h +++ b/include/sys/zia.h @@ -155,7 +155,7 @@ int zia_offload_abd(void *provider, abd_t *abd, size_t size, size_t min_offload_size, boolean_t *local_offload, boolean_t lock); int zia_onload_abd(abd_t *abd, size_t size, - boolean_t keep_handle, boolean_t lock); + boolean_t keep_handle); /* move a handle into an abd */ void zia_move_into_abd(abd_t *dst, void **src); int zia_free_abd(abd_t *abd, boolean_t lock); diff --git a/module/os/linux/zfs/vdev_disk.c b/module/os/linux/zfs/vdev_disk.c index 7e4988055622..3fd338215aa2 100644 --- a/module/os/linux/zfs/vdev_disk.c +++ b/module/os/linux/zfs/vdev_disk.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -793,6 +794,8 @@ __vdev_disk_physio(struct block_device *bdev, zio_t *zio, return (error); } +EXPORT_SYMBOL(__vdev_disk_physio); + BIO_END_IO_PROTO(vdev_disk_io_flush_completion, bio, error) { zio_t *zio = bio->bi_private; @@ -835,6 +838,8 @@ vdev_disk_io_flush(struct block_device *bdev, zio_t *zio) return (0); } +EXPORT_SYMBOL(vdev_disk_io_flush); + static int vdev_disk_io_trim(zio_t *zio) { diff --git a/module/os/linux/zfs/vdev_file.c b/module/os/linux/zfs/vdev_file.c index 09e175cf9deb..96f591dd8110 100644 --- a/module/os/linux/zfs/vdev_file.c +++ b/module/os/linux/zfs/vdev_file.c @@ -36,6 +36,7 @@ #include #include #include +#include #ifdef _KERNEL #include #endif @@ -161,6 +162,12 @@ vdev_file_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize, } #endif + zia_get_props(vd->vdev_spa)->min_offload_size = 2 << *physical_ashift; + + /* try to open the file; ignore errors - will fall back to ZFS */ + zia_file_open(vd, vd->vdev_path, + vdev_file_open_mode(spa_mode(vd->vdev_spa)), 0); + skip_open: error = zfs_file_getattr(vf->vf_file, &zfa); @@ -184,6 +191,8 @@ vdev_file_close(vdev_t *vd) if (vd->vdev_reopening || vf == NULL) return; + zia_file_close(vd); + if (vf->vf_file != NULL) { (void) zfs_file_close(vf->vf_file); } @@ -203,18 +212,53 @@ vdev_file_io_strategy(void *arg) void *buf; loff_t off; ssize_t size; - int err; + int err = 0; off = zio->io_offset; size = zio->io_size; resid = 0; if (zio->io_type == ZIO_TYPE_READ) { - buf = abd_borrow_buf(zio->io_abd, zio->io_size); + buf = abd_borrow_buf(zio->io_abd, size); err = zfs_file_pread(vf->vf_file, buf, size, off, &resid); abd_return_buf_copy(zio->io_abd, buf, size); } else { - buf = abd_borrow_buf_copy(zio->io_abd, zio->io_size); + err = EIO; + + boolean_t local_offload = B_FALSE; + zia_props_t *zia_props = zia_get_props(zio->io_spa); + + if ((zia_props->file_write == 1) && + (zio->io_can_offload == B_TRUE)) { + if (zia_offload_abd(zia_props->provider, zio->io_abd, + size, zia_props->min_offload_size, + &local_offload, B_TRUE) == ZIA_OK) { + err = zia_file_write(vd, zio->io_abd, size, off, + &resid, &err); + } + } + + /* if offload and write succeeded, return here */ + if (err == 0) { + zio->io_error = err; + if (resid != 0 && zio->io_error == 0) + zio->io_error = SET_ERROR(ENOSPC); + + zio_delay_interrupt(zio); + return; + } + + /* if offload or write failed, bring data back into memory */ + err = zia_cleanup_abd(zio->io_abd, size, local_offload, B_TRUE); + + /* if onload failed, restart the zio with offloading disabled */ + if (err == ZIA_ACCELERATOR_DOWN) { + zia_disable_offloading(zio, B_TRUE); + zio_delay_interrupt(zio); + return; + } + + buf = abd_borrow_buf_copy(zio->io_abd, size); err = zfs_file_pwrite(vf->vf_file, buf, size, off, &resid); abd_return_buf(zio->io_abd, buf, size); } diff --git a/module/zfs/vdev_raidz.c b/module/zfs/vdev_raidz.c index b0a655220882..ad5ad943021b 100644 --- a/module/zfs/vdev_raidz.c +++ b/module/zfs/vdev_raidz.c @@ -521,6 +521,8 @@ vdev_raidz_generate_parity_p(raidz_row_t *rr) } } +EXPORT_SYMBOL(vdev_raidz_generate_parity_p); + void vdev_raidz_generate_parity_pq(raidz_row_t *rr) { @@ -563,6 +565,8 @@ vdev_raidz_generate_parity_pq(raidz_row_t *rr) } } +EXPORT_SYMBOL(vdev_raidz_generate_parity_pq); + void vdev_raidz_generate_parity_pqr(raidz_row_t *rr) { @@ -611,6 +615,8 @@ vdev_raidz_generate_parity_pqr(raidz_row_t *rr) } } +EXPORT_SYMBOL(vdev_raidz_generate_parity_pqr); + /* * Generate RAID parity in the first virtual columns according to the number of * parity columns available. @@ -1417,6 +1423,8 @@ vdev_raidz_reconstruct_general(raidz_row_t *rr, int *tgts, int ntgts) } } +EXPORT_SYMBOL(vdev_raidz_reconstruct_general); + static void vdev_raidz_reconstruct_row(raidz_map_t *rm, raidz_row_t *rr, const int *t, int nt) diff --git a/module/zfs/zia.c b/module/zfs/zia.c index 769b8bf85868..33e4f7ccbfcd 100644 --- a/module/zfs/zia.c +++ b/module/zfs/zia.c @@ -679,7 +679,7 @@ zia_offload_abd(void *provider, abd_t *abd, size_t size, size_t min_offload_size, boolean_t *local_offload, boolean_t lock) { - if (!dpusm) { + if (!dpusm || !provider) { return (ZIA_FALLBACK); } @@ -687,18 +687,18 @@ zia_offload_abd(void *provider, abd_t *abd, return (ZIA_ERROR); } - if (lock) { - mutex_enter(&abd->abd_mtx); - } + if (lock) { + mutex_enter(&abd->abd_mtx); + } const int rc = zia_offload_abd_offset(provider, abd, 0, size, min_offload_size, local_offload); - if (lock) { - mutex_exit(&abd->abd_mtx); - } + if (lock) { + mutex_exit(&abd->abd_mtx); + } - return (rc); + return (rc); } #ifdef ZIA @@ -779,7 +779,7 @@ zia_onload_abd_offset(abd_t *abd, size_t offset, } int -zia_onload_abd(abd_t *abd, size_t size, boolean_t keep_handle, boolean_t lock) +zia_onload_abd(abd_t *abd, size_t size, boolean_t keep_handle) { if (abd_is_gang(abd)) { /* @@ -803,17 +803,7 @@ zia_onload_abd(abd_t *abd, size_t size, boolean_t keep_handle, boolean_t lock) ASSERT(size <= original_size); } - if (lock == B_TRUE) { - mutex_enter(&abd->abd_mtx); - } - - const int err = zia_onload_abd_offset(abd, 0, size, keep_handle); - - if (lock == B_TRUE) { - mutex_exit(&abd->abd_mtx); - } - - return (err); + return (zia_onload_abd_offset(abd, 0, size, keep_handle)); } void @@ -843,7 +833,8 @@ zia_free_abd(abd_t *abd, boolean_t lock) * if not, onload the data and free the handle */ int -zia_cleanup_abd(abd_t *abd, size_t size, boolean_t local_offload, boolean_t lock) +zia_cleanup_abd(abd_t *abd, size_t size, + boolean_t local_offload, boolean_t lock) { if (!dpusm) { return (ZIA_FALLBACK); @@ -854,13 +845,22 @@ zia_cleanup_abd(abd_t *abd, size_t size, boolean_t local_offload, boolean_t lock } int ret = ZIA_OK; + + if (lock == B_TRUE) { + mutex_enter(&abd->abd_mtx); + } + if (local_offload == B_TRUE) { /* in-memory copy is still valid */ /* lock just in case mirrors clean up at the same time */ - ret = zia_free_abd(abd, B_TRUE); + ret = zia_free_abd(abd, B_FALSE); } else { /* have to copy data into memory */ - ret = zia_onload_abd(abd, size, B_FALSE, lock); + ret = zia_onload_abd(abd, size, B_FALSE); + } + + if (lock == B_TRUE) { + mutex_exit(&abd->abd_mtx); } return (ret); @@ -1020,7 +1020,7 @@ zia_checksum_compute(void *provider, zio_cksum_t *dst, enum zio_checksum alg, if (zia_offload_abd(provider, zio->io_abd, size, zia_get_props(zio->io_spa)->min_offload_size, - local_offload, B_FALSE) != ZIA_OK) { + local_offload, B_FALSE) != ZIA_OK) { return (ZIA_ERROR); } } else { @@ -1247,7 +1247,7 @@ zia_raidz_alloc(zio_t *zio, raidz_row_t *rr, boolean_t rec, */ zia_offload_abd(provider, rc->rc_abd, rc->rc_abd->abd_size, props->min_offload_size, - NULL, B_FALSE); + NULL, B_FALSE); handle = ABD_HANDLE(rc->rc_abd); } else { /* generating - create new columns */ @@ -1352,7 +1352,7 @@ zia_raidz_free(raidz_row_t *rr, boolean_t onload_parity) raidz_col_t *rc = &rr->rr_col[c]; ret = zia_worst_error(ret, zia_onload_abd(rc->rc_abd, - rc->rc_size, B_FALSE, B_FALSE)); + rc->rc_size, B_FALSE)); } } @@ -1519,10 +1519,10 @@ zia_file_open(vdev_t *vdev, const char *path, return (ZIA_ERROR); } - zia_props_t *zia_props = zia_get_props(vdev->vdev_spa); - if (zia_props->file_write != 1) { - return (ZIA_FALLBACK); - } + zia_props_t *zia_props = zia_get_props(vdev->vdev_spa); + if (zia_props->file_write != 1) { + return (ZIA_FALLBACK); + } #ifdef ZIA void *provider = zia_props->provider; diff --git a/module/zfs/zio.c b/module/zfs/zio.c index 3eb472a9fd2a..0db93a3a662b 100644 --- a/module/zfs/zio.c +++ b/module/zfs/zio.c @@ -51,6 +51,7 @@ #include #include #include +#include #include /* @@ -394,12 +395,12 @@ zio_push_transform(zio_t *zio, abd_t *data, uint64_t size, uint64_t bufsize, zio->io_size = size; } -void -zio_pop_transforms(zio_t *zio) +zio_transform_t * +zio_pop_transform(zio_t *zio) { - zio_transform_t *zt; + zio_transform_t *zt = zio->io_transform_stack; - while ((zt = zio->io_transform_stack) != NULL) { + if (zt != NULL) { if (zt->zt_transform != NULL) zt->zt_transform(zio, zt->zt_orig_abd, zt->zt_orig_size); @@ -413,6 +414,15 @@ zio_pop_transforms(zio_t *zio) kmem_free(zt, sizeof (zio_transform_t)); } + + return (zt); +} + +void +zio_pop_transforms(zio_t *zio) +{ + while (zio_pop_transform(zio)) { + } } /* @@ -433,11 +443,42 @@ static void zio_decompress(zio_t *zio, abd_t *data, uint64_t size) { if (zio->io_error == 0) { - void *tmp = abd_borrow_buf(data, size); - int ret = zio_decompress_data(BP_GET_COMPRESS(zio->io_bp), - zio->io_abd, tmp, zio->io_size, size, - &zio->io_prop.zp_complevel); - abd_return_buf_copy(data, tmp, size); + int ret = ZIA_FALLBACK; + zia_props_t *zia_props = zia_get_props(zio->io_spa); + if ((zia_props->decompress == 1) && + (zio->io_can_offload == B_TRUE)) { + ret = zia_decompress(zia_props, + BP_GET_COMPRESS(zio->io_bp), + zio->io_abd, zio->io_size, + data, size, + &zio->io_prop.zp_complevel); + } + + if (ret == ZIA_OK) { + ASSERT(zia_is_offloaded(zio->io_abd) == B_TRUE); + /* + * bring data back into memory since there + * are no subsequent offloaded stages + */ + ret = zia_onload_abd(data, size, B_FALSE); + } + + ASSERT(zia_is_offloaded(data) != B_TRUE); + /* let abd_free clean up zio->io_abd */ + + if (ret == ZIA_OK) { + ret = 0; + } else { + if (ret == ZIA_ACCELERATOR_DOWN) { + zia_disable_offloading(zio, B_FALSE); + } + + void *tmp = abd_borrow_buf(data, size); + ret = zio_decompress_data(BP_GET_COMPRESS(zio->io_bp), + zio->io_abd, tmp, zio->io_size, size, + &zio->io_prop.zp_complevel); + abd_return_buf_copy(data, tmp, size); + } if (zio_injection_enabled && ret == 0) ret = zio_handle_fault_injection(zio, EINVAL); @@ -748,6 +789,11 @@ zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait, if (zio->io_error && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) *errorp = zio_worst_error(*errorp, zio->io_error); pio->io_reexecute |= zio->io_reexecute; + if ((zio->io_flags & ZIO_FLAG_ZIA_REEXECUTE) && + (zio->io_can_offload != B_TRUE)) { + pio->io_flags |= ZIO_FLAG_ZIA_REEXECUTE; + pio->io_can_offload = B_FALSE; + } ASSERT3U(*countp, >, 0); (*countp)--; @@ -798,6 +844,10 @@ zio_inherit_child_errors(zio_t *zio, enum zio_child c) { if (zio->io_child_error[c] != 0 && zio->io_error == 0) zio->io_error = zio->io_child_error[c]; + + if (zio->io_flags & ZIO_FLAG_ZIA_REEXECUTE) { + zio->io_can_offload = B_FALSE; + } } int @@ -918,7 +968,13 @@ zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, if (zb != NULL) zio->io_bookmark = *zb; + zio->io_can_offload = zia_get_props(spa)->can_offload; + if (pio != NULL) { + if ((pio->io_flags & ZIO_FLAG_ZIA_REEXECUTE) || + (pio->io_can_offload != B_TRUE)) { + zio->io_can_offload = B_FALSE; + } zio->io_metaslab_class = pio->io_metaslab_class; if (zio->io_logical == NULL) zio->io_logical = pio->io_logical; @@ -927,6 +983,13 @@ zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, zio_add_child_first(pio, zio); } + /* turn off encryption and dedup if Z.I.A. is used */ + if (zia_is_used(zio) == B_TRUE) { + zio->io_prop.zp_dedup = B_FALSE; + zio->io_prop.zp_dedup_verify = B_FALSE; + zio->io_prop.zp_encrypt = B_FALSE; + } + taskq_init_ent(&zio->io_tqent); return (zio); @@ -1811,18 +1874,85 @@ zio_write_compress(zio_t *zio) if (compress != ZIO_COMPRESS_OFF && !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) { void *cbuf = NULL; - psize = zio_compress_data(compress, zio->io_abd, &cbuf, lsize, - zp->zp_complevel); + int zia_rc = ZIA_FALLBACK; + void *cbuf_handle = NULL; /* only valid if zia_rc == ZIA_OK */ + zia_props_t *zia_props = zia_get_props(spa); + boolean_t local_offload = B_FALSE; + if ((zia_props->compress == 1) && + (zio->io_can_offload == B_TRUE)) { + zia_rc = zia_compress(zia_props, compress, + zio->io_abd, lsize, &cbuf_handle, &psize, + zp->zp_complevel, &local_offload); + } + + if (zia_rc != ZIA_OK) { + ASSERT(cbuf_handle == NULL); + zia_rc = zia_cleanup_abd(zio->io_abd, + lsize, local_offload, B_FALSE); + + /* + * if data has to be brought back for cpu compression, + * but could not, restart the pipeline for this zio + * (not necessary in this case, but still doing it here + * in case a previous stage is offloaded) + */ + if (zia_rc == ZIA_ACCELERATOR_DOWN) { + zia_restart_before_vdev(zio); + return (zio); + } + psize = zio_compress_data(compress, zio->io_abd, &cbuf, + lsize, zp->zp_complevel); + } + if (psize == 0) { compress = ZIO_COMPRESS_OFF; + ASSERT0(cbuf_handle); } else if (psize >= lsize) { compress = ZIO_COMPRESS_OFF; if (cbuf != NULL) zio_buf_free(cbuf, lsize); + /* + * no need for offloaded + * compressed buffer any more + * + * catch accelerator failures elsewhere + */ + zia_free(&cbuf_handle); + + /* source abd is still offloaded */ } else if (!zp->zp_dedup && !zp->zp_encrypt && psize <= BPE_PAYLOAD_SIZE && zp->zp_level == 0 && !DMU_OT_HAS_FILL(zp->zp_type) && spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA)) { + + /* + * compressed enough, but not handling embedded + * data, so move compressed data back into memory + */ + if (cbuf_handle) { + cbuf = zio_buf_alloc(lsize); + } + + if (zia_onload(&cbuf_handle, cbuf, + psize) == ZIA_ACCELERATOR_DOWN) { + zia_free_abd(zio->io_abd, B_FALSE); + zio_buf_free(cbuf, lsize); + zia_restart_before_vdev(zio); + return (zio); + } + + /* + * remove offloaded source abd + * + * in-memory copy should still be valid, but calling + * zia_cleanup_abd just in case + */ + if (zia_cleanup_abd(zio->io_abd, lsize, + local_offload, B_FALSE) == ZIA_ACCELERATOR_DOWN) { + zio_buf_free(cbuf, lsize); + zia_restart_before_vdev(zio); + return (zio); + } encode_embedded_bp_compressed(bp, cbuf, compress, lsize, psize); BPE_SET_ETYPE(bp, BP_EMBEDDED_TYPE_DATA); @@ -1847,15 +1977,68 @@ zio_write_compress(zio_t *zio) psize); if (rounded >= lsize) { compress = ZIO_COMPRESS_OFF; - zio_buf_free(cbuf, lsize); + if (cbuf_handle) { + /* + * catch accelerator + * down elsewhere + */ + zia_free(&cbuf_handle); + } else { + zio_buf_free(cbuf, lsize); + } psize = lsize; } else { + /* abd_get_from_buf must not get a NULL */ + if (cbuf_handle) { + cbuf = zio_buf_alloc(lsize); + } abd_t *cdata = abd_get_from_buf(cbuf, lsize); abd_take_ownership_of_buf(cdata, B_TRUE); + if (cbuf_handle) { + /* + * zio->io_abd offload no longer needed + * but don't free here - let abd_free + * handle it + */ + + /* + * compressed enough, so associate the + * compressed buffer with the abd + */ + zia_move_into_abd(cdata, &cbuf_handle); + zia_rc = zia_zero_fill(cdata, psize, + rounded - psize); + if (zia_rc != ZIA_OK) { + /* + * if setting cdata's handle + * fails, onload the compressed + * buffer (automatically placing + * it into cdata) and continue + * using zfs + * + * if cbuf is not offloaded, + * nothing happens + */ + zia_rc = zia_onload( + &cbuf_handle, cbuf, lsize); + } + + if (zia_rc == ZIA_ACCELERATOR_DOWN) { + zia_free(&cbuf_handle); + zia_free_abd(zio->io_abd, + B_FALSE); + zia_restart_before_vdev(zio); + return (zio); + } + } abd_zero_off(cdata, psize, rounded - psize); psize = rounded; zio_push_transform(zio, cdata, psize, lsize, NULL); + if (zia_is_offloaded(zio->io_abd)) { + zio->io_flags |= + ZIO_FLAG_DONT_AGGREGATE; + } } } @@ -3924,6 +4107,13 @@ zio_vdev_io_start(zio_t *zio) if (zio->io_type == ZIO_TYPE_WRITE) { abd_copy(abuf, zio->io_abd, zio->io_size); abd_zero_off(abuf, zio->io_size, asize - zio->io_size); + /* + * The Z.I.A. handles of the abds that come here + * were not modified and do not get associated with + * abuf during the transform. Instead of dropping + * the handle and delaying here, let abd_free clean + * it up later. + */ } zio_push_transform(zio, abuf, asize, asize, zio_subblock); } @@ -4114,6 +4304,8 @@ zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr) { void *abd = abd_alloc_sametype(zio->io_abd, zio->io_size); + zia_onload_abd(zio->io_abd, zio->io_size, B_FALSE); + abd_copy(abd, zio->io_abd, zio->io_size); zcr->zcr_cbinfo = zio->io_size; @@ -4148,7 +4340,9 @@ zio_vdev_io_assess(zio_t *zio) * On retry, we cut in line in the issue queue, since we don't want * compression/checksumming/etc. work to prevent our (cheap) IO reissue. */ - if (zio->io_error && vd == NULL && + if (zio->io_error && + !(zio->io_flags & ZIO_FLAG_ZIA_REEXECUTE) && + vd == NULL && !(zio->io_flags & (ZIO_FLAG_DONT_RETRY | ZIO_FLAG_IO_RETRY))) { ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_QUEUE)); /* not a leaf */ ASSERT(!(zio->io_flags & ZIO_FLAG_IO_BYPASS)); /* not a leaf */ @@ -4773,6 +4967,8 @@ zio_done(zio_t *zio) } if (zio->io_error) { + ASSERT(!(zio->io_flags & ZIO_FLAG_ZIA_REEXECUTE)); + /* * If this I/O is attached to a particular vdev, * generate an error message describing the I/O failure @@ -4807,7 +5003,10 @@ zio_done(zio_t *zio) } } - if (zio->io_error && zio == zio->io_logical) { + if ((zio->io_error || + (zio->io_flags & ZIO_FLAG_ZIA_REEXECUTE) || + 0) && + zio == zio->io_logical) { /* * Determine whether zio should be reexecuted. This will * propagate all the way to the root via zio_notify_parent(). @@ -5179,6 +5378,7 @@ EXPORT_SYMBOL(zio_buf_alloc); EXPORT_SYMBOL(zio_data_buf_alloc); EXPORT_SYMBOL(zio_buf_free); EXPORT_SYMBOL(zio_data_buf_free); +EXPORT_SYMBOL(zio_push_transform); ZFS_MODULE_PARAM(zfs_zio, zio_, slow_io_ms, INT, ZMOD_RW, "Max I/O completion time (milliseconds) before marking it as slow"); diff --git a/module/zfs/zio_checksum.c b/module/zfs/zio_checksum.c index 9de515e8767a..ae38f2423cf6 100644 --- a/module/zfs/zio_checksum.c +++ b/module/zfs/zio_checksum.c @@ -31,6 +31,7 @@ #include #include #include +#include #include /* @@ -357,6 +358,13 @@ zio_checksum_compute(zio_t *zio, enum zio_checksum checksum, zio_eck_t eck; size_t eck_offset; + /* not handling embedded checksums, so bring back data */ + const int zia_rc = zia_cleanup_abd(abd, size, B_FALSE, B_FALSE); + if (zia_rc == ZIA_ACCELERATOR_DOWN) { + zia_restart_before_vdev(zio); + return; + } + memset(&saved, 0, sizeof (zio_cksum_t)); if (checksum == ZIO_CHECKSUM_ZILOG2) { @@ -400,8 +408,31 @@ zio_checksum_compute(zio_t *zio, enum zio_checksum checksum, sizeof (zio_cksum_t)); } else { saved = bp->blk_cksum; + + int zia_rc = ZIA_FALLBACK; + + /* only offload non-embedded checksums */ + boolean_t local_offload = B_FALSE; + zia_props_t *zia_props = zia_get_props(spa); + if ((zia_props->checksum == 1) && + (zio->io_can_offload == B_TRUE)) { + zia_rc = zia_checksum_compute(zia_props->provider, + &cksum, checksum, zio, size, &local_offload); + } + + /* fall back to ZFS implementation */ + if (zia_rc != ZIA_OK) { + zia_rc = zia_cleanup_abd(abd, size, local_offload, + B_FALSE); + if (zia_rc == ZIA_ACCELERATOR_DOWN) { + zia_restart_before_vdev(zio); + return; + } ci->ci_func[0](abd, size, spa->spa_cksum_tmpls[checksum], &cksum); + } else { + zio->io_flags |= ZIO_FLAG_DONT_AGGREGATE; + } if (BP_USES_CRYPT(bp) && BP_GET_TYPE(bp) != DMU_OT_OBJSET) zio_checksum_handle_crypt(&cksum, &saved, insecure); bp->blk_cksum = cksum; @@ -430,6 +461,12 @@ zio_checksum_error_impl(spa_t *spa, const blkptr_t *bp, zio_cksum_t verifier; size_t eck_offset; + /* not handling embedded checksums, so bring back data */ + const int zia_rc = zia_cleanup_abd(abd, size, B_FALSE, B_FALSE); + if (zia_rc == ZIA_ACCELERATOR_DOWN) { + return (zia_rc); + } + if (checksum == ZIO_CHECKSUM_ZILOG2) { zil_chain_t zilc; uint64_t nused; @@ -490,8 +527,25 @@ zio_checksum_error_impl(spa_t *spa, const blkptr_t *bp, } else { byteswap = BP_SHOULD_BYTESWAP(bp); expected_cksum = bp->blk_cksum; - ci->ci_func[byteswap](abd, size, - spa->spa_cksum_tmpls[checksum], &actual_cksum); + + zia_props_t *zia_props = zia_get_props(spa); + int error = ZIA_FALLBACK; + if ((zia_props->can_offload == B_TRUE) && + (zia_props->checksum == 1)) { + error = zia_checksum_error(checksum, abd, size, + byteswap, &actual_cksum); + } + + /* fall back to ZFS implementation */ + if ((error != ZIA_OK) && (error != ECKSUM)) { + /* data was modified by reconstruction */ + error = zia_onload_abd(abd, size, B_FALSE); + if (error == ZIA_ACCELERATOR_DOWN) { + return (error); + } + ci->ci_func[byteswap](abd, size, + spa->spa_cksum_tmpls[checksum], &actual_cksum); + } } /*