diff --git a/cmd/zstream/scripts/add-xattrs.py b/cmd/zstream/scripts/add-xattrs.py new file mode 100755 index 000000000000..12b2483a364e --- /dev/null +++ b/cmd/zstream/scripts/add-xattrs.py @@ -0,0 +1,91 @@ +#!/tmp/zstream-venv/bin/python3 +"""Add random extended attributes to files until 600 bytes of xattrs are added.""" + +import argparse +import os +import random +import sys +from lorem_text import lorem + +ADJECTIVES = [ + "boogie", "funky", "wobbly", "snazzy", "jazzy", "groovy", "zippy", + "bouncy", "fluffy", "crunchy", "sparkly", "fuzzy", "spiffy", "dandy", + "peppy", "snappy", "sassy", "zesty", "swanky", "nifty", "plucky", + "quirky", "wacky", "goofy", "dizzy", "breezy", "cheery", "perky", + "frisky", "chirpy", "feisty", "jolly", "lively", "merry", "spunky", + "zippy", "vivid", "brisk", "sunny", "witty", "kinky", +] + +NOUNS = [ + "woogie", "monkey", "noodle", "pickle", "muffin", "waffle", "pebble", + "wobble", "doodle", "tangle", "giggle", "wiggle", "jiggle", "sparkle", + "crinkle", "twinkle", "frizzle", "drizzle", "sizzle", "fizzle", + "puddle", "bubble", "muddle", "huddle", "cuddle", "juggle", "muggle", + "snuggle", "tuggle", "buggle", "nugget", "widget", "gadget", "gibbet", + "trinket", "bracket", "racket", "jacket", "ticket", "cricket", "thicket", + "biscuit", "circuit", "summit", "muppet", "trumpet", "basket", "casket", +] + +TARGET_BYTES = 1024 + + +def random_attr_name(used: set) -> str: + for _ in range(1000): + name = f"user.{random.choice(ADJECTIVES)}-{random.choice(NOUNS)}" + if name not in used: + return name + base = f"user.{random.choice(ADJECTIVES)}-{random.choice(NOUNS)}" + i = 2 + while f"{base}-{i}" in used: + i += 1 + return f"{base}-{i}" + + +def random_value(length: int) -> bytes: + # Pull words from lorem sentences and trim/pad to exact length + text = "" + while len(text) < length: + text += lorem.sentence() + " " + return text[:length].encode() + + +def add_xattrs(path: str) -> int: + """Add xattrs to path until TARGET_BYTES total value bytes added. Returns bytes added.""" + used_names = set() + total = 0 + while total < TARGET_BYTES: + remaining = TARGET_BYTES - total + length = min(random.randint(40, 200), remaining) if remaining < 40 else random.randint(40, min(200, remaining)) + # If remaining < 40, just do one final attr to hit the target + if remaining < 40: + length = remaining + name = random_attr_name(used_names) + used_names.add(name) + value = random_value(length) + os.setxattr(path, name, value) + total += len(value) + return total + + +def main(): + parser = argparse.ArgumentParser( + description=f"Add random xattrs to files until {TARGET_BYTES} bytes of xattr values are added." + ) + parser.add_argument("files", nargs="+", help="Files to annotate with xattrs") + args = parser.parse_args() + + errors = 0 + for path in args.files: + try: + added = add_xattrs(path) + print(f" {path} ({added:,} bytes in xattrs)") + except OSError as e: + print(f" {path} error: {e}", file=sys.stderr) + errors += 1 + + if errors: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/cmd/zstream/scripts/gen-lorem-files.py b/cmd/zstream/scripts/gen-lorem-files.py new file mode 100755 index 000000000000..84ea1faf6386 --- /dev/null +++ b/cmd/zstream/scripts/gen-lorem-files.py @@ -0,0 +1,83 @@ +#!/tmp/zstream-venv/bin/python3 +"""Generate files with random names filled with lorem ipsum paragraphs.""" + +import argparse +import random +import sys +from pathlib import Path +from lorem_text import lorem + +ADJECTIVES = [ + "boogie", "funky", "wobbly", "snazzy", "jazzy", "groovy", "zippy", + "bouncy", "fluffy", "crunchy", "sparkly", "fuzzy", "spiffy", "dandy", + "peppy", "snappy", "sassy", "zesty", "swanky", "nifty", "plucky", + "quirky", "wacky", "goofy", "dizzy", "breezy", "cheery", "perky", + "frisky", "chirpy", "feisty", "jolly", "lively", "merry", "spunky", + "frisky", "zippy", "vivid", "brisk", "sunny", "witty", "kinky", +] + +NOUNS = [ + "woogie", "monkey", "noodle", "pickle", "muffin", "waffle", "pebble", + "wobble", "doodle", "tangle", "giggle", "wiggle", "jiggle", "sparkle", + "crinkle", "twinkle", "frizzle", "drizzle", "sizzle", "fizzle", + "puddle", "bubble", "muddle", "huddle", "cuddle", "juggle", "muggle", + "snuggle", "tuggle", "buggle", "nugget", "widget", "gadget", "gibbet", + "trinket", "bracket", "racket", "jacket", "ticket", "cricket", "thicket", + "biscuit", "circuit", "summit", "muppet", "trumpet", "basket", "casket", +] + +def random_name(used: set) -> str: + for _ in range(1000): + name = f"{random.choice(ADJECTIVES)}-{random.choice(NOUNS)}" + if name not in used: + return name + # Fallback: append a number + base = f"{random.choice(ADJECTIVES)}-{random.choice(NOUNS)}" + i = 2 + while f"{base}-{i}" in used: + i += 1 + return f"{base}-{i}" + + +def fill_file(path: Path, target_size: int, repeat=False) -> None: + content_parts = [] + total = 0 + para = lorem.paragraph() + while total < target_size: + content_parts.append(para) + total += len(para) + 1 # +1 for newline + if not repeat: + para = lorem.paragraph() + path.write_text("\n\n".join(content_parts) + "\n") + + +def main(): + parser = argparse.ArgumentParser( + description="Generate files with random names and lorem ipsum content." + ) + parser.add_argument("count", type=int, help="Number of files to create") + parser.add_argument("-d", "--directory", default=".", help="Target directory (default: .)") + parser.add_argument("-r", "--repeat", action="store_true", help="Fill files with reps of a single paragraph") + parser.add_argument("--min-size", type=int, default=16384, help="Minimum file size in bytes (default: 2048)") + parser.add_argument("--max-size", type=int, default=128000, help="Maximum file size in bytes (default: 128000)") + args = parser.parse_args() + + if args.min_size >= args.max_size: + print(f"error: min-size ({args.min_size}) must be less than max-size ({args.max_size})", file=sys.stderr) + sys.exit(1) + + directory = Path(args.directory) + directory.mkdir(parents=True, exist_ok=True) + + used_names = set() + for i in range(args.count): + name = random_name(used_names) + used_names.add(name) + target_size = random.randint(args.min_size, args.max_size) + path = directory / name + fill_file(path, target_size, args.repeat) + print(f" {path} ({path.stat().st_size:,} bytes)") + + +if __name__ == "__main__": + main() diff --git a/cmd/zstream/scripts/make-all-records-streams.sh b/cmd/zstream/scripts/make-all-records-streams.sh new file mode 100755 index 000000000000..e518625beea0 --- /dev/null +++ b/cmd/zstream/scripts/make-all-records-streams.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +if [ $# -ne 1 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +DEVICE="$1" +SCRIPTDIR="$(cd "$(dirname "$0")" && pwd)" + +zpool create -o ashift=12 test "$DEVICE" +zfs set compression=on xattr=sa test +zfs create test/source + +"$SCRIPTDIR/gen-lorem-files.py" -r -d /test/source --min-size 2048 \ + --max-size 32000 3 +"$SCRIPTDIR/add-xattrs.py" /test/source/* +echo "very small" > /test/source/small +echo "password" > /test/source/to-be-redacted +chmod 400 /test/source/to-be-redacted + +zfs snapshot -r test/source@baseline +zfs clone test/source@baseline test/redacted +rm /test/redacted/to-be-redacted +"$SCRIPTDIR/gen-lorem-files.py" -r -d /test/redacted --min-size 4096 \ + --max-size 32000 3 +"$SCRIPTDIR/add-xattrs.py" /test/redacted/* +cd /test/redacted +tar cf /tmp/dups.tar . +mkdir copies +cd copies +tar xvf /tmp/dups.tar + +echo "password" > /test/redacted/new-key +zfs create -o encryption=on -o keylocation=file:///test/redacted/new-key \ + -o keyformat=passphrase test/redacted/encrypted +"$SCRIPTDIR/gen-lorem-files.py" -r -d /test/redacted/encrypted 3 +echo "very small" > /test/redacted/encrypted/small-encrypted +# "$SCRIPTDIR/add-xattrs.py" /test/redacted/encrypted/* + +zfs snapshot -r test/redacted@clean + +zfs redact test/source@baseline redaction-bookmark test/redacted@clean +zfs send -ce --redact redaction-bookmark test/source@baseline > /tmp/all-record-types-base.zsend +zfs send -Rcew -i test/source@baseline test/redacted@clean > /tmp/all-record-types-incr.zsend diff --git a/cmd/zstream/scripts/make-decompression-streams.sh b/cmd/zstream/scripts/make-decompression-streams.sh new file mode 100755 index 000000000000..2eba3c52889c --- /dev/null +++ b/cmd/zstream/scripts/make-decompression-streams.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +if [ $# -ne 1 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +DEVICE="$1" +SCRIPTDIR="$(cd "$(dirname "$0")" && pwd)" + +zpool create -o ashift=12 test "$DEVICE" +echo "password" > /test/password + +zfs create -o compression=zstd-5 test/unencrypted +"$SCRIPTDIR/gen-lorem-files.py" -r -d /test/unencrypted --min-size 12000 \ + --max-size 40000 2 +"$SCRIPTDIR/gen-lorem-files.py" -r -d /test/unencrypted --min-size 140000 \ + --max-size 160000 1 + +zfs create -o compression=lz4 -o encryption=on -o keylocation=file:///test/password -o keyformat=passphrase test/encrypted +"$SCRIPTDIR/gen-lorem-files.py" -r -d /test/encrypted --min-size 12000 \ + --max-size 40000 3 + +zfs snapshot -r test@decompression +zfs send -cw test/unencrypted@decompression > /tmp/decompression.zsend +zfs send -cw test/encrypted@decompression > /tmp/decompression-crypt.zsend diff --git a/cmd/zstream/scripts/make-dump-files.py b/cmd/zstream/scripts/make-dump-files.py new file mode 100755 index 000000000000..49fbef54bb3e --- /dev/null +++ b/cmd/zstream/scripts/make-dump-files.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Run old and new zstream dump -v on stream files, producing abbreviated dump outputs.""" +import argparse +import subprocess +import sys +from pathlib import Path + +def abbreviate(filename: str) -> str: + """Split filename at dashes, take the first letter of each segment, lowercased.""" + stem = Path(filename).stem + # Strip common compression suffixes to get the logical stem + for ext in (".zfs", ".gz", ".bz2", ".xz", ".zst", ".lz4"): + if stem.endswith(ext): + stem = stem[: -len(ext)] + return "".join(seg[0] for seg in stem.split("-") if seg).lower() + +def run_dump(zstream: Path, stream: Path, output: Path) -> bool: + """Run `zstream dump -v < stream > output`. Returns True on success.""" + try: + with open(stream, "rb") as inf, open(output, "w") as outf: + proc = subprocess.run( + [str(zstream), "dump", "-v"], + stdin=inf, + stdout=outf, + stderr=outf, + ) + if proc.returncode != 0: + print( + f" WARNING: {zstream} exited {proc.returncode} for {stream.name}", + file=sys.stderr, + ) + if proc.stderr: + print(f" stderr: {proc.stderr.decode(errors='replace').rstrip()}", + file=sys.stderr) + return True + except Exception as e: + print(f" ERROR: {e}", file=sys.stderr) + return False + + +def main(): + parser = argparse.ArgumentParser( + description="Run old and new zstream dump -v on stream files." + ) + parser.add_argument("old_zstream", type=Path, help="Path to old zstream binary") + parser.add_argument("new_zstream", type=Path, help="Path to new zstream binary") + parser.add_argument( + "streams", nargs="+", type=Path, help="Compressed stream files to process" + ) + args = parser.parse_args() + + for zs in (args.old_zstream, args.new_zstream): + if not zs.is_file(): + parser.error(f"zstream binary not found: {zs}") + + for stream in args.streams: + if not stream.is_file(): + print(f"Skipping missing file: {stream}", file=sys.stderr) + continue + + abbrev = abbreviate(stream.name) + out_dir = stream.parent + + old_out = out_dir / f"{abbrev}-old.dump" + new_out = out_dir / f"{abbrev}-new.dump" + + print(f"{stream.name} -> {abbrev}") + + print(f" old: {old_out}") + run_dump(args.old_zstream, stream, old_out) + + print(f" new: {new_out}") + run_dump(args.new_zstream, stream, new_out) + + +if __name__ == "__main__": + main() diff --git a/cmd/zstream/scripts/make-long-payloads.sh b/cmd/zstream/scripts/make-long-payloads.sh new file mode 100755 index 000000000000..2a5251111601 --- /dev/null +++ b/cmd/zstream/scripts/make-long-payloads.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +if [ $# -ne 1 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +DEVICE="$1" +SCRIPTDIR="$(cd "$(dirname "$0")" && pwd)" + +zpool create -o ashift=12 test "$DEVICE" + +zfs set compression=off recordsize=16MiB test + +# We are testing 8MB blocks, so write one short file, 8.5MB +# file, and one 24.5MB file. + +"$SCRIPTDIR/gen-lorem-files.py" -d /test -r --min-size 20000 \ + --max-size 24000 1 +"$SCRIPTDIR/gen-lorem-files.py" -d /test -r --min-size 8500000 \ + --max-size 8510000 1 +"$SCRIPTDIR/gen-lorem-files.py" -d /test -r --min-size 24500000 \ + --max-size 24510000 1 + +zfs snapshot test@long-payloads +zfs send -L test@long-payloads > /tmp/long-payloads.zsend diff --git a/cmd/zstream/scripts/make-venv.sh b/cmd/zstream/scripts/make-venv.sh new file mode 100755 index 000000000000..ced816dcb3d8 --- /dev/null +++ b/cmd/zstream/scripts/make-venv.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +python3 -m venv /tmp/zstream-venv +. /tmp/zstream-venv/bin/activate +pip install lorem_text diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index 14e4bd79f853..9435b366d32e 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -1003,7 +1003,7 @@ tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos', 'rsend_030_pos', 'rsend_031_pos', 'rsend-exclude_001_pos', 'rsend-exclude_002_pos', 'send-c_verify_ratio', 'send-c_verify_contents', 'send-c_props', 'send-c_incremental', - 'send-c_volume', 'send-c_zstream_recompress', 'send-c_zstreamdump', + 'send-c_volume', 'send-c_lz4_disabled', 'send-c_recv_lz4_disabled', 'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize', @@ -1012,7 +1012,7 @@ tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos', 'send_encrypted_props', 'send_encrypted_truncated_files', 'send_freeobjects', 'send_realloc_files', 'send_realloc_encrypted_files', 'send_spill_block', 'send_holds', 'send_hole_birth', 'send_mixed_raw', - 'send-wR_encrypted_zvol', 'send-zstream_drop_record', + 'send-wR_encrypted_zvol', 'send_partial_dataset', 'send_invalid', 'send_large_blocks_incremental', 'send_large_blocks_initial', 'send_large_microzap_incremental', 'send_large_microzap_transitive', @@ -1131,6 +1131,20 @@ tests = ['zoned_uid_001_pos', 'zoned_uid_002_pos', 'zoned_uid_003_pos', 'zoned_uid_029_neg', 'zoned_uid_031_pos'] tags = ['functional', 'zoned_uid'] +[tests/functional/zstream] +tests = ['zstream_checksum_001_pos', + 'zstream_decompress_001_pos', 'zstream_decompress_002_pos', + 'zstream_decompress_003_neg', 'zstream_decompress_004_pos', + 'zstream_decompress_005_pos', 'zstream_decompress_006_neg', + 'zstream_drop_record_001_pos', + 'zstream_dump_001_pos', 'zstream_dump_002_pos', + 'zstream_dump_003_pos', 'zstream_dump_004_neg', + 'zstream_recompress_001_pos', 'zstream_recompress_002_pos', + 'zstream_recompress_003_pos', 'zstream_recompress_004_pos', + 'zstream_recompress_005_pos', + 'zstream_redup_001_pos'] +tags = ['functional', 'zstream'] + [tests/functional/zvol/zvol_ENOSPC] tests = ['zvol_ENOSPC_001_pos'] tags = ['functional', 'zvol', 'zvol_ENOSPC'] diff --git a/tests/runfiles/sanity.run b/tests/runfiles/sanity.run index 936f2bcc32be..2f42fcb4b638 100644 --- a/tests/runfiles/sanity.run +++ b/tests/runfiles/sanity.run @@ -561,7 +561,7 @@ tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos', 'rsend_006_pos', 'rsend_009_pos', 'rsend_010_pos', 'rsend_011_pos', 'rsend_014_pos', 'rsend_016_neg', 'rsend-exclude_001_pos', 'rsend-exclude_002_pos', 'send-c_verify_contents', - 'send-c_volume', 'send-c_zstreamdump', 'send-c_recv_dedup', + 'send-c_volume', 'send-c_recv_dedup', 'send-L_toggle', 'send_encrypted_hierarchy', 'send_encrypted_props', 'send_encrypted_freeobjects', 'send_encrypted_truncated_files', 'send_freeobjects', 'send_holds', @@ -629,6 +629,11 @@ tests = ['xattr_001_pos', 'xattr_002_neg', 'xattr_003_neg', 'xattr_004_pos', 'xattr_011_pos', 'xattr_013_pos', 'xattr_014_pos', 'xattr_compat'] tags = ['functional', 'xattr'] +[tests/functional/zstream] +tests = ['zstream_dump_001_pos', 'zstream_dump_002_pos', + 'zstream_redup_001_pos'] +tags = ['functional', 'zstream'] + [tests/functional/zvol/zvol_ENOSPC] tests = ['zvol_ENOSPC_001_pos'] tags = ['functional', 'zvol', 'zvol_ENOSPC'] diff --git a/tests/zfs-tests/tests/Makefile.am b/tests/zfs-tests/tests/Makefile.am index 28acc6f3af12..d416a444a90d 100644 --- a/tests/zfs-tests/tests/Makefile.am +++ b/tests/zfs-tests/tests/Makefile.am @@ -401,6 +401,42 @@ nobase_dist_datadir_zfs_tests_tests_DATA += \ functional/zvol/zvol_ENOSPC/zvol_ENOSPC.cfg \ functional/zvol/zvol_misc/zvol_misc_common.kshlib \ functional/zvol/zvol_swap/zvol_swap.cfg \ + functional/zstream/zstream.cfg \ + functional/zstream/zstream.kshlib \ + functional/zstream/big-endian-all-drr-types-base-NATIVE.zsend.bz2 \ + functional/zstream/big-endian-all-drr-types-base-XDR.zsend.bz2 \ + functional/zstream/big-endian-all-drr-types-incr-NATIVE.zsend.bz2 \ + functional/zstream/big-endian-all-drr-types-incr-XDR.zsend.bz2 \ + functional/zstream/little-endian-all-drr-types-base-NATIVE.zsend.bz2 \ + functional/zstream/little-endian-all-drr-types-base-XDR.zsend.bz2 \ + functional/zstream/little-endian-all-drr-types-incr-NATIVE.zsend.bz2 \ + functional/zstream/little-endian-all-drr-types-incr-XDR.zsend.bz2 \ + functional/zstream/decompress.zsend.bz2 \ + functional/zstream/decompress-crypt.zsend.bz2 \ + functional/zstream/big-endian-long-payloads.zsend.bz2 \ + functional/zstream/little-endian-long-payloads.zsend.bz2 \ + functional/zstream/beadtbn-new.dump.bz2 \ + functional/zstream/beadtbn-old.dump.bz2 \ + functional/zstream/beadtbx-new.dump.bz2 \ + functional/zstream/beadtbx-old.dump.bz2 \ + functional/zstream/beadtin-new.dump.bz2 \ + functional/zstream/beadtin-old.dump.bz2 \ + functional/zstream/beadtix-new.dump.bz2 \ + functional/zstream/beadtix-old.dump.bz2 \ + functional/zstream/belp-new.dump.bz2 \ + functional/zstream/belp-old.dump.bz2 \ + functional/zstream/d-new.dump.bz2 \ + functional/zstream/dc-new.dump.bz2 \ + functional/zstream/leadtbn-new.dump.bz2 \ + functional/zstream/leadtbn-old.dump.bz2 \ + functional/zstream/leadtbx-new.dump.bz2 \ + functional/zstream/leadtbx-old.dump.bz2 \ + functional/zstream/leadtin-new.dump.bz2 \ + functional/zstream/leadtin-old.dump.bz2 \ + functional/zstream/leadtix-new.dump.bz2 \ + functional/zstream/leadtix-old.dump.bz2 \ + functional/zstream/lelp-new.dump.bz2 \ + functional/zstream/lelp-old.dump.bz2 \ functional/idmap_mount/idmap_mount.cfg \ functional/idmap_mount/idmap_mount_common.kshlib @@ -2091,8 +2127,6 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/rsend/send-c_verify_contents.ksh \ functional/rsend/send-c_verify_ratio.ksh \ functional/rsend/send-c_volume.ksh \ - functional/rsend/send-c_zstream_recompress.ksh \ - functional/rsend/send-c_zstreamdump.ksh \ functional/rsend/send-cpL_varied_recsize.ksh \ functional/rsend/send_doall.ksh \ functional/rsend/send_encrypted_incremental.ksh \ @@ -2121,7 +2155,6 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/rsend/send_realloc_files.ksh \ functional/rsend/send_spill_block.ksh \ functional/rsend/send-wR_encrypted_zvol.ksh \ - functional/rsend/send-zstream_drop_record.ksh \ functional/rsend/setup.ksh \ functional/scrub_mirror/cleanup.ksh \ functional/scrub_mirror/scrub_mirror_001_pos.ksh \ @@ -2330,6 +2363,26 @@ nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \ functional/zpool_influxdb/cleanup.ksh \ functional/zpool_influxdb/setup.ksh \ functional/zpool_influxdb/zpool_influxdb.ksh \ + functional/zstream/setup.ksh \ + functional/zstream/cleanup.ksh \ + functional/zstream/zstream_checksum_001_pos.ksh \ + functional/zstream/zstream_decompress_001_pos.ksh \ + functional/zstream/zstream_decompress_002_pos.ksh \ + functional/zstream/zstream_decompress_003_neg.ksh \ + functional/zstream/zstream_decompress_004_pos.ksh \ + functional/zstream/zstream_decompress_005_pos.ksh \ + functional/zstream/zstream_decompress_006_neg.ksh \ + functional/zstream/zstream_drop_record_001_pos.ksh \ + functional/zstream/zstream_dump_001_pos.ksh \ + functional/zstream/zstream_dump_002_pos.ksh \ + functional/zstream/zstream_dump_003_pos.ksh \ + functional/zstream/zstream_dump_004_neg.ksh \ + functional/zstream/zstream_recompress_001_pos.ksh \ + functional/zstream/zstream_recompress_002_pos.ksh \ + functional/zstream/zstream_recompress_003_pos.ksh \ + functional/zstream/zstream_recompress_004_pos.ksh \ + functional/zstream/zstream_recompress_005_pos.ksh \ + functional/zstream/zstream_redup_001_pos.ksh \ functional/zvol/zvol_cli/cleanup.ksh \ functional/zvol/zvol_cli/setup.ksh \ functional/zvol/zvol_cli/zvol_cli_001_pos.ksh \ diff --git a/tests/zfs-tests/tests/functional/zstream/beadtbn-new.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/beadtbn-new.dump.bz2 new file mode 100644 index 000000000000..947d615c7462 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/beadtbn-new.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/beadtbn-old.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/beadtbn-old.dump.bz2 new file mode 100644 index 000000000000..c80330c6e736 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/beadtbn-old.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/beadtbx-new.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/beadtbx-new.dump.bz2 new file mode 100644 index 000000000000..884bbde5fcad Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/beadtbx-new.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/beadtbx-old.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/beadtbx-old.dump.bz2 new file mode 100644 index 000000000000..b52fc091e705 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/beadtbx-old.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/beadtin-new.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/beadtin-new.dump.bz2 new file mode 100644 index 000000000000..22b4ed986a18 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/beadtin-new.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/beadtin-old.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/beadtin-old.dump.bz2 new file mode 100644 index 000000000000..e467af8ce1bc Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/beadtin-old.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/beadtix-new.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/beadtix-new.dump.bz2 new file mode 100644 index 000000000000..456d6da63547 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/beadtix-new.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/beadtix-old.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/beadtix-old.dump.bz2 new file mode 100644 index 000000000000..8a677e3d030b Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/beadtix-old.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/belp-new.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/belp-new.dump.bz2 new file mode 100644 index 000000000000..49a98d9c62af Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/belp-new.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/belp-old.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/belp-old.dump.bz2 new file mode 100644 index 000000000000..83a20115a842 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/belp-old.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-base-NATIVE.zsend.bz2 b/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-base-NATIVE.zsend.bz2 new file mode 100644 index 000000000000..5ebe9e40c1cd Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-base-NATIVE.zsend.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-base-XDR.zsend.bz2 b/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-base-XDR.zsend.bz2 new file mode 100644 index 000000000000..a9c6c0445346 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-base-XDR.zsend.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-incr-NATIVE.zsend.bz2 b/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-incr-NATIVE.zsend.bz2 new file mode 100644 index 000000000000..385aecd46aa2 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-incr-NATIVE.zsend.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-incr-XDR.zsend.bz2 b/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-incr-XDR.zsend.bz2 new file mode 100644 index 000000000000..162dde833c91 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/big-endian-all-drr-types-incr-XDR.zsend.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/big-endian-long-payloads.zsend.bz2 b/tests/zfs-tests/tests/functional/zstream/big-endian-long-payloads.zsend.bz2 new file mode 100644 index 000000000000..3fae565450b6 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/big-endian-long-payloads.zsend.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/cleanup.ksh b/tests/zfs-tests/tests/functional/zstream/cleanup.ksh new file mode 100755 index 000000000000..e63e0553cec3 --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/cleanup.ksh @@ -0,0 +1,39 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026 by ConnectWise. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +verify_runnable "both" + +if is_global_zone ; then + destroy_pool $POOL +else + cleanup_pool $POOL +fi +log_must rm -rf $BACKDIR + +log_pass diff --git a/tests/zfs-tests/tests/functional/zstream/d-new.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/d-new.dump.bz2 new file mode 100644 index 000000000000..a378b6c2aef9 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/d-new.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/dc-new.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/dc-new.dump.bz2 new file mode 100644 index 000000000000..613ffbd3b3bc Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/dc-new.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/decompress-crypt.zsend.bz2 b/tests/zfs-tests/tests/functional/zstream/decompress-crypt.zsend.bz2 new file mode 100644 index 000000000000..de1931ff6bc7 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/decompress-crypt.zsend.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/decompress.zsend.bz2 b/tests/zfs-tests/tests/functional/zstream/decompress.zsend.bz2 new file mode 100644 index 000000000000..7f2ef6141cdc Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/decompress.zsend.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/leadtbn-new.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/leadtbn-new.dump.bz2 new file mode 100644 index 000000000000..753082a111b2 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/leadtbn-new.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/leadtbn-old.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/leadtbn-old.dump.bz2 new file mode 100644 index 000000000000..7838243c9cf3 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/leadtbn-old.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/leadtbx-new.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/leadtbx-new.dump.bz2 new file mode 100644 index 000000000000..70862d375d9b Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/leadtbx-new.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/leadtbx-old.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/leadtbx-old.dump.bz2 new file mode 100644 index 000000000000..48c4892c181a Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/leadtbx-old.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/leadtin-new.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/leadtin-new.dump.bz2 new file mode 100644 index 000000000000..b25ed3897ada Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/leadtin-new.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/leadtin-old.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/leadtin-old.dump.bz2 new file mode 100644 index 000000000000..fd9acbf8507c Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/leadtin-old.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/leadtix-new.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/leadtix-new.dump.bz2 new file mode 100644 index 000000000000..b3de0d7bda00 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/leadtix-new.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/leadtix-old.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/leadtix-old.dump.bz2 new file mode 100644 index 000000000000..04d0c41b9978 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/leadtix-old.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/lelp-new.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/lelp-new.dump.bz2 new file mode 100644 index 000000000000..67a3eb068806 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/lelp-new.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/lelp-old.dump.bz2 b/tests/zfs-tests/tests/functional/zstream/lelp-old.dump.bz2 new file mode 100644 index 000000000000..fce108ee3e0e Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/lelp-old.dump.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-base-NATIVE.zsend.bz2 b/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-base-NATIVE.zsend.bz2 new file mode 100644 index 000000000000..de08809d7945 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-base-NATIVE.zsend.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-base-XDR.zsend.bz2 b/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-base-XDR.zsend.bz2 new file mode 100644 index 000000000000..bb8ab79fcf60 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-base-XDR.zsend.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-incr-NATIVE.zsend.bz2 b/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-incr-NATIVE.zsend.bz2 new file mode 100644 index 000000000000..ec781bc96ec8 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-incr-NATIVE.zsend.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-incr-XDR.zsend.bz2 b/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-incr-XDR.zsend.bz2 new file mode 100644 index 000000000000..73cea6f1ed6b Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/little-endian-all-drr-types-incr-XDR.zsend.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/little-endian-long-payloads.zsend.bz2 b/tests/zfs-tests/tests/functional/zstream/little-endian-long-payloads.zsend.bz2 new file mode 100644 index 000000000000..16ca84407051 Binary files /dev/null and b/tests/zfs-tests/tests/functional/zstream/little-endian-long-payloads.zsend.bz2 differ diff --git a/tests/zfs-tests/tests/functional/zstream/setup.ksh b/tests/zfs-tests/tests/functional/zstream/setup.ksh new file mode 100755 index 000000000000..6bf896479046 --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/setup.ksh @@ -0,0 +1,37 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026 by ConnectWise. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +verify_runnable "both" + +if is_global_zone ; then + log_must zpool create $POOL $DISK1 +fi +log_must mkdir -p $BACKDIR + +log_pass diff --git a/tests/zfs-tests/tests/functional/zstream/zstream.cfg b/tests/zfs-tests/tests/functional/zstream/zstream.cfg new file mode 100644 index 000000000000..58f8b7e43440 --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream.cfg @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026 by ConnectWise. All rights reserved. +# + +export BACKDIR=${TEST_BASE_DIR%%/}/backdir-zstream +export PATH=/usr/local/sbin:/usr/local/bin:/usr/bin:${PATH} + +read -r DISK1 _ <<<"$DISKS" +export DISK1 + +export POOL=$TESTPOOL diff --git a/tests/zfs-tests/tests/functional/zstream/zstream.kshlib b/tests/zfs-tests/tests/functional/zstream/zstream.kshlib new file mode 100644 index 000000000000..917e01b6b33b --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream.kshlib @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: CDDL-1.0 +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +# or https://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at usr/src/OPENSOLARIS.LICENSE. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# + +# +# Copyright (c) 2026 by ConnectWise. All rights reserved. +# + +. $STF_SUITE/include/libtest.shlib +. $STF_SUITE/include/math.shlib +. $STF_SUITE/tests/functional/zstream/zstream.cfg + +# +# Several zstream tests need functions defined in rsend.kshlib +# (stream_has_features, get_resume_token, cleanup_pool). Rather than +# duplicate them, source the rsend library. Note that rsend.kshlib +# also sources rsend.cfg, which will define its own $BACKDIR, $POOL, +# $POOL2, etc. Our zstream.cfg is sourced first, and we reassert our +# values afterward so that our tests use the zstream-specific paths +# and pool names. +# +. $STF_SUITE/tests/functional/rsend/rsend.kshlib + +# Reassert our config over rsend's +. $STF_SUITE/tests/functional/zstream/zstream.cfg + +ZSTREAM_DATADIR=$STF_SUITE/tests/functional/zstream + +# +# Return "little" or "big" depending on the system's byte order. +# +function get_system_endian +{ + python3 -c "import sys; print(sys.byteorder)" +} + +# +# Map a stream filename stem to its dump-file abbreviation. +# e.g. "big-endian-all-drr-types-base-NATIVE" -> "beadtbn" +# +function get_stream_abbrev +{ + typeset stem=$1 + typeset -l abbrev="" + typeset IFS="-" + + for part in $stem; do + abbrev="${abbrev}${part%"${part#?}"}" + done + + echo "$abbrev" +} + +# +# Receive a stream into $POOL/recv, compute sorted xxhsum -H2 of all +# regular files under the mountpoint, and write the result to $1. +# Destroys $POOL/recv afterward if $2 is set to "cleanup". +# +function recv_and_hash +{ + typeset hashfile=$1 + typeset stream=$2 + typeset cleanup=${3:-""} + + log_must zfs receive -F $POOL/recv < "$stream" + typeset mnt=$(get_prop mountpoint $POOL/recv) + ( cd "$mnt" && find . -type f -print0 | sort -z | \ + xargs -0 xxhsum -H2 ) > "$hashfile" + if [[ -n $cleanup ]]; then + log_must zfs destroy -r $POOL/recv + fi +} diff --git a/tests/zfs-tests/tests/functional/zstream/zstream_checksum_001_pos.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_checksum_001_pos.ksh new file mode 100755 index 000000000000..4238c178be26 --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream_checksum_001_pos.ksh @@ -0,0 +1,66 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that very long payload records are checksummed correctly by +# running them through zstream redup (which recalculates checksums but is +# otherwise a no-op). This exercises the 8MiB Fletcher4 chunk boundary +# handling. There are test streams of both endiannesses because opposite- +# endian checksumming is a slightly separate path. +# +# Strategy: +# 1. Decompress the long-payloads test streams +# 2. Pipe through zstream redup +# 3. Verify the output is byte-identical to the input +# + +verify_runnable "both" + +log_assert "Verify long payload records are checksummed correctly." + +typeset -a streams=( + little-endian-long-payloads + big-endian-long-payloads +) + +typeset failed="" + +for stem in "${streams[@]}"; do + + typeset src="$ZSTREAM_DATADIR/${stem}.zsend.bz2" + typeset orig="$BACKDIR/${stem}.zsend.orig" + typeset redup_out="$BACKDIR/${stem}.zsend.redup" + + bzcat "$src" > "$orig" + log_must eval "zstream redup '$orig' > '$redup_out'" + + if cmp -s "$ref" "$out" > /dev/null 2>&1; then + log_note "MISMATCH: $stem" + failed="$failed $stem" + fi + + log_must cmp -s "$orig" "$redup_out" + +done + +[[ -z $failed ]] || log_fail "Round-trip mismatch for: $failed" + +log_pass "Long-payload records are checksummed correctly." diff --git a/tests/zfs-tests/tests/functional/zstream/zstream_decompress_001_pos.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_decompress_001_pos.ksh new file mode 100755 index 000000000000..9029b4880619 --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream_decompress_001_pos.ksh @@ -0,0 +1,98 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream decompress actually decompresses selected WRITE +# records. The input stream contains zstd-compressed writes. +# +# Strategy: +# 1. Decompress selected records (2,0 3,0 128,131072) from the stream +# 2. Run zstream dump -v on both original and decompressed streams +# 3. Verify the selected records now show compression type = 0, +# compressed_size = 0, and payload_size = logical_size +# + +verify_runnable "both" + +log_assert "Verify zstream decompress decompresses selected WRITE records." + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/decompress.orig.zsend" +typeset decompressed="$BACKDIR/decompress.out.zsend" +typeset orig_dump="$BACKDIR/decompress.orig.dump" +typeset decomp_dump="$BACKDIR/decompress.out.dump" + +# Selected records: object,offset +typeset -a records=(2,0 3,0 128,131072) + +# Decompress the bz2 and run zstream decompress on selected records +bzcat "$src" > "$orig" +log_must eval "zstream decompress ${records[*]} < '$orig' > '$decompressed'" + +# Dump both streams +log_must eval "zstream dump -v < '$orig' > '$orig_dump' 2>&1" +log_must eval "zstream dump -v < '$decompressed' > '$decomp_dump' 2>&1" + +# For each selected record, verify it was decompressed in the output +typeset failed="" +for rec in "${records[@]}"; do + typeset obj=${rec%,*} + typeset off=${rec#*,} + + # Find the WRITE line for this object/offset in the original dump + # and extract the logical_size + typeset orig_line=$(awk \ + "/^WRITE object = $obj .* offset = $off /" \ + "$orig_dump") + typeset lsize=$(echo "$orig_line" | \ + sed 's/.*logical_size = \([0-9]*\).*/\1/') + + # Find the same record in the decompressed dump + typeset decomp_line=$(awk \ + "/^WRITE object = $obj .* offset = $off /" \ + "$decomp_dump") + + # Verify compression type = 0 + if ! echo "$decomp_line" | grep -q 'compression type = 0'; then + log_note "Record $rec: compression type not 0" + log_note " got: $decomp_line" + failed="$failed ${rec}(comp)" + fi + + # Verify compressed_size = 0 (indicates uncompressed) + if ! echo "$decomp_line" | grep -q 'compressed_size = 0'; then + log_note "Record $rec: compressed_size not 0" + failed="$failed ${rec}(csize)" + fi + + # Verify payload_size = logical_size + typeset psize=$(echo "$decomp_line" | \ + sed 's/.*payload_size = \([0-9]*\).*/\1/') + if [[ "$psize" != "$lsize" ]]; then + log_note "Record $rec: payload_size ($psize) != logical_size ($lsize)" + failed="$failed ${rec}(psize)" + fi +done + +[[ -z $failed ]] || \ + log_fail "Decompression verification failed for:$failed" + +log_pass "zstream decompress decompresses selected WRITE records." diff --git a/tests/zfs-tests/tests/functional/zstream/zstream_decompress_002_pos.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_decompress_002_pos.ksh new file mode 100755 index 000000000000..1c7d9b7dcfc0 --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream_decompress_002_pos.ksh @@ -0,0 +1,57 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that a decompressed stream produces identical filesystem contents +# when received. +# +# Strategy: +# 1. Receive the original decompress.zsend into a test pool and hash files +# 2. Receive the decompressed stream (with records 2,0 3,0 +# and 128,131072 decompressed) +# 3. Verify file hashes are identical +# + +verify_runnable "both" + +log_assert "Verify decompressed stream receives with identical file contents." +log_onexit cleanup_pool $POOL + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/decompress.orig.zsend" +typeset decompressed="$BACKDIR/decompress.out.zsend" + +typeset -a records=(2,0 3,0 128,131072) + +# Prepare streams +bzcat "$src" > "$orig" +log_must eval "zstream decompress ${records[*]} < '$orig' > '$decompressed'" + +# Receive original and hash +recv_and_hash "$BACKDIR/hash-orig.txt" "$orig" cleanup + +# Receive decompressed and hash +recv_and_hash "$BACKDIR/hash-decomp.txt" "$decompressed" cleanup + +# Compare +log_must diff "$BACKDIR/hash-orig.txt" "$BACKDIR/hash-decomp.txt" + +log_pass "Decompressed stream receives with identical file contents." diff --git a/tests/zfs-tests/tests/functional/zstream/zstream_decompress_003_neg.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_decompress_003_neg.ksh new file mode 100755 index 000000000000..9a394d14f972 --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream_decompress_003_neg.ksh @@ -0,0 +1,59 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that specifying the wrong compression type (lz4 for a zstd +# stream) causes decompression to fail gracefully, leaving the output +# stream identical to the input. +# +# Strategy: +# 1. Run zstream decompress with lz4 type on zstd-compressed records +# 2. Verify the output stream is byte-identical to the input +# 3. Check stderr for failure messages +# + +verify_runnable "both" + +log_assert "Verify wrong compression type leaves stream unchanged." + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/decompress.orig" +typeset output="$BACKDIR/decompress-lz4.out" +typeset errfile="$BACKDIR/decompress-lz4.err" + +typeset -a records=(2,0,lz4 3,0,lz4 128,131072,lz4) + +bzcat "$src" > "$orig" + +# Attempt to decompress zstd records as lz4 — should fail for each +zstream decompress ${records[*]} < "$orig" > "$output" 2>"$errfile" + +# Output stream must be identical to input (nothing decompressed) +log_must cmp -s "$orig" "$output" + +# Stderr should contain messages about the failed decompressions +typeset errcount=$(wc -l < "$errfile") +if [[ $errcount -ne 3 ]]; then + log_fail "Did not receive 3 error messages on stderr, got $errcount" +fi +log_note "Got $errcount lines of error output (expected)" + +log_pass "Wrong compression type leaves stream unchanged." diff --git a/tests/zfs-tests/tests/functional/zstream/zstream_decompress_004_pos.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_decompress_004_pos.ksh new file mode 100755 index 000000000000..9b16d89cc17e --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream_decompress_004_pos.ksh @@ -0,0 +1,55 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that specifying the correct compression type (zstd) produces +# the same output as omitting the type. +# +# Strategy: +# 1. Decompress records without specifying type +# 2. Decompress records specifying zstd as type +# 3. Verify both outputs are identical +# + +verify_runnable "both" + +log_assert "Verify explicit correct compression type matches default." + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/decompress.orig" +typeset out_default="$BACKDIR/decompress-default.out" +typeset out_zstd="$BACKDIR/decompress-zstd.out" + +typeset -a records=(2,0 3,0 128,131072) +typeset -a zstd_records=(2,0,zstd 3,0,zstd 128,131072,zstd) + +bzcat "$src" > "$orig" + +# Decompress without specifying type +log_must eval "zstream decompress ${records[*]} < '$orig' > '$out_default'" + +# Decompress specifying zstd +log_must eval "zstream decompress ${zstd_records[*]} < '$orig' > '$out_zstd'" + +# Both outputs must be identical +log_must cmp -s "$out_default" "$out_zstd" + +log_pass "Explicit correct compression type matches default." diff --git a/tests/zfs-tests/tests/functional/zstream/zstream_decompress_005_pos.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_decompress_005_pos.ksh new file mode 100755 index 000000000000..b5dfb518ae24 --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream_decompress_005_pos.ksh @@ -0,0 +1,119 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream decompress with "off" as the compression type +# changes record headers to mark them as uncompressed but leaves the +# actual data payload untouched. This means the received files will +# contain raw compressed data (junk) for the affected records. +# +# Strategy: +# 1. Decompress selected records with type "off" +# 2. Verify via zstream dump that selected records now show +# compression type = 0 and logical_size is unchanged +# 3. Receive both original and "off" streams into a test pool +# 4. Verify that file hashes differ (junk data in affected files) +# + +verify_runnable "both" + +log_assert "Verify decompress with 'off' changes headers but not data." +log_onexit cleanup_pool $POOL + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/decompress.orig" +typeset off_out="$BACKDIR/decompress-off.out" +typeset orig_dump="$BACKDIR/decompress-orig.dump" +typeset off_dump="$BACKDIR/decompress-off.dump" + +typeset -a records=(2,0 3,0 128,131072) +typeset -a off_records=(2,0,off 3,0,off 128,131072,off) + +bzcat "$src" > "$orig" + +log_must eval "zstream decompress ${off_records[*]} < '$orig' > '$off_out'" + +# Dump both streams +log_must eval "zstream dump -v < '$orig' > '$orig_dump' 2>&1" +log_must eval "zstream dump -v < '$off_out' > '$off_dump' 2>&1" + +# Verify selected records show compression type = 0 with same logical_size +typeset failed="" +for rec in "${records[@]}"; do + typeset obj=${rec%,*} + typeset off=${rec#*,} + + typeset orig_line=$(awk \ + "/^WRITE object = $obj .* offset = $off /" \ + "$orig_dump") + typeset orig_lsize=$(echo "$orig_line" | \ + sed 's/.*logical_size = \([0-9]*\).*/\1/') + typeset orig_csize=$(echo "$orig_line" | \ + sed 's/.*compressed_size = \([0-9]*\).*/\1/') + typeset orig_ctype=$(echo "$orig_line" | \ + sed 's/.*compression type = \([0-9]*\).*/\1/') + + typeset off_line=$(awk \ + "/^WRITE object = $obj .* offset = $off /" \ + "$off_dump") + typeset off_lsize=$(echo "$off_line" | \ + sed 's/.*logical_size = \([0-9]*\).*/\1/') + typeset off_csize=$(echo "$off_line" | \ + sed 's/.*compressed_size = \([0-9]*\).*/\1/') + typeset off_ctype=$(echo "$off_line" | \ + sed 's/.*compression type = \([0-9]*\).*/\1/') + + if [[ "$orig_ctype" == "0" ]]; then + log_note "Record $rec: original compression type is 0" + failed="$failed ${rec}(orig ctype)" + fi + if [[ "$off_ctype" != "0" ]]; then + log_note "Record $rec: modified compression type is not 0" + failed="$failed ${rec}(modified ctype)" + fi + if [[ "$off_lsize" != "$orig_csize" ]]; then + log_note "Record $rec: modified logical_size != original " \ + "compressed_size ($orig_lsize -> $off_lsize)" + failed="$failed ${rec}(lsize)" + fi + if [[ "$off_csize" != "0" ]]; then + log_note "Record $rec: modified compressed_size != 0 " \ + "($orig_csize -> $off_csize)" + failed="$failed ${rec}(csize)" + fi +done + +[[ -z $failed ]] || \ + log_fail "Header verification failed for:$failed" + +# Receive original and hash +recv_and_hash "$BACKDIR/hash-orig.txt" "$orig" cleanup + +# Receive "off" stream and hash +recv_and_hash "$BACKDIR/hash-off.txt" "$off_out" cleanup + +# Hashes must differ (affected files now contain junk data) +if diff -q "$BACKDIR/hash-orig.txt" "$BACKDIR/hash-off.txt" \ + > /dev/null 2>&1; then + log_fail "Expected file contents to differ, but hashes are identical" +fi + +log_pass "Decompress with 'off' changes headers but not data." diff --git a/tests/zfs-tests/tests/functional/zstream/zstream_decompress_006_neg.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_decompress_006_neg.ksh new file mode 100755 index 000000000000..32ce347e4afa --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream_decompress_006_neg.ksh @@ -0,0 +1,61 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream decompress prints a warning to stderr for encrypted +# WRITES, attempts to decompress anyway, fails, and leaves stream unchanged. +# +# Strategy: +# 1. Select compressed encrypted records from decompress-crypt.zsend +# 2. Attempt to decompress them +# 3. Verify stderr contains warnings for each record +# 4. Verify output stream is byte-identical to input +# + +verify_runnable "both" + +log_assert "Verify that zstream decompress handles encrypted records correctly." + +typeset src="$ZSTREAM_DATADIR/decompress-crypt.zsend.bz2" +typeset orig="$BACKDIR/decompress-crypt.orig" +typeset output="$BACKDIR/decompress-crypt.out" +typeset errfile="$BACKDIR/decompress-crypt.err" + + + +typeset -a records=(2,0 3,0 36,0) + +bzcat "$src" > "$orig" + +# Attempt to decompress encrypted records +zstream decompress ${records[*]} < "$orig" > "$output" 2>"$errfile" + +# Output stream must be identical to input +log_must cmp -s "$orig" "$output" + +# Stderr should contain warnings about each record +typeset errcount=$(wc -l < "$errfile") +if [[ $errcount -ne 6 ]]; then + log_fail "Expected 6 messages on stderr, got $errcount" +fi +log_note "Got $errcount lines of warning output (expected)" + +log_pass "Encrypted records refuse to decompress." diff --git a/tests/zfs-tests/tests/functional/rsend/send-zstream_drop_record.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_drop_record_001_pos.ksh similarity index 93% rename from tests/zfs-tests/tests/functional/rsend/send-zstream_drop_record.ksh rename to tests/zfs-tests/tests/functional/zstream/zstream_drop_record_001_pos.ksh index a2e810fd40dc..25ffcb3015c7 100755 --- a/tests/zfs-tests/tests/functional/rsend/send-zstream_drop_record.ksh +++ b/tests/zfs-tests/tests/functional/zstream/zstream_drop_record_001_pos.ksh @@ -16,8 +16,7 @@ # Copyright (c) 2026 by ConnectWise. All rights reserved. # -. $STF_SUITE/tests/functional/rsend/rsend.kshlib -. $STF_SUITE/include/math.shlib +. $STF_SUITE/tests/functional/zstream/zstream.kshlib # # Description: @@ -32,10 +31,10 @@ verify_runnable "both" log_assert "Verify zstream drop_record correctly drops records." -log_onexit cleanup_pool $POOL2 +log_onexit cleanup_pool $POOL -typeset sendfs=$POOL2/fs -typeset recvfs=$POOL2/fs2 +typeset sendfs=$POOL/fs +typeset recvfs=$POOL/fs2 typeset stream=$BACKDIR/stream typeset filtered=$BACKDIR/filtered typeset dump=$BACKDIR/dump diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_zstreamdump.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_dump_001_pos.ksh similarity index 90% rename from tests/zfs-tests/tests/functional/rsend/send-c_zstreamdump.ksh rename to tests/zfs-tests/tests/functional/zstream/zstream_dump_001_pos.ksh index 5ff2cbf0c6a9..c604f36c52fa 100755 --- a/tests/zfs-tests/tests/functional/rsend/send-c_zstreamdump.ksh +++ b/tests/zfs-tests/tests/functional/zstream/zstream_dump_001_pos.ksh @@ -17,12 +17,11 @@ # Copyright (c) 2020 by Datto, Inc. All rights reserved. # -. $STF_SUITE/tests/functional/rsend/rsend.kshlib -. $STF_SUITE/include/math.shlib +. $STF_SUITE/tests/functional/zstream/zstream.kshlib # # Description: -# Verify compression features show up in zstream dump +# Verify that compression features show up in zstream dump # # Strategy: # 1. Create a full compressed send stream @@ -36,11 +35,11 @@ verify_runnable "both" log_assert "Verify zstream dump correctly interprets compressed send streams." -log_onexit cleanup_pool $POOL2 +log_onexit cleanup_pool $POOL -typeset sendfs=$POOL2/fs -typeset streamfs=$POOL2/fs2 -typeset recvfs=$POOL2/fs3 +typeset sendfs=$POOL/fs +typeset streamfs=$POOL/fs2 +typeset recvfs=$POOL/fs3 log_must zfs create -o compress=lz4 $sendfs log_must zfs create -o compress=lz4 $streamfs diff --git a/tests/zfs-tests/tests/functional/zstream/zstream_dump_002_pos.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_dump_002_pos.ksh new file mode 100755 index 000000000000..558a79a29659 --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream_dump_002_pos.ksh @@ -0,0 +1,84 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream dump -v output is as expected for pregenerated +# same-endian, neutral (no BEGIN nvlists), and XDR-encoded test streams. +# NV_ENCODE_NATIVE-encoded BEGIN records aren't readable on opposite-endian +# systems, so for these the output of zstream dump varies according to +# host endianness. +# +# Strategy: +# 1. For each of the test streams, run zstream dump -v +# 2. Compare stdout+stderr with the corresponding reference dump +# + +verify_runnable "both" + +log_assert "Verify zstream dump -v output matches reference dump files." + +typeset sys_endian=$(get_system_endian) + +typeset -a streams=( + decompress + decompress-crypt + little-endian-long-payloads + big-endian-long-payloads + big-endian-all-drr-types-base-XDR + big-endian-all-drr-types-incr-XDR + little-endian-all-drr-types-base-XDR + little-endian-all-drr-types-incr-XDR +) + +if [[ $sys_endian == "little" ]]; then + streams+=( + little-endian-all-drr-types-base-NATIVE + little-endian-all-drr-types-incr-NATIVE + ) +else + streams+=( + big-endian-all-drr-types-base-NATIVE + big-endian-all-drr-types-incr-NATIVE + ) +fi + +typeset failed="" + +for stem in "${streams[@]}"; do + typeset abbrev=$(get_stream_abbrev "$stem") + typeset ref_src="$ZSTREAM_DATADIR/${abbrev}-new.dump.bz2" + typeset send_src="$ZSTREAM_DATADIR/${stem}.zsend.bz2" + typeset ref="$BACKDIR/${abbrev}-new.dump" + typeset out="$BACKDIR/${abbrev}-out.dump" + + bzcat "$send_src" | zstream dump -v > "$out" 2>&1 + bzcat "$ref_src" > "$ref" + + if ! diff -q "$ref" "$out" > /dev/null 2>&1; then + log_note "MISMATCH: $stem (abbrev $abbrev)" + log_note "$(diff "$ref" "$out")" + failed="$failed $stem" + fi +done + +[[ -z $failed ]] || log_fail "Dump output mismatch for:$failed" + +log_pass "zstream dump -v output matches reference dump files." diff --git a/tests/zfs-tests/tests/functional/zstream/zstream_dump_003_pos.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_dump_003_pos.ksh new file mode 100755 index 000000000000..edcfe877a93b --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream_dump_003_pos.ksh @@ -0,0 +1,96 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream dump -v output for all same-endian and XDR-encoded +# test streams matches that of the previous version of zstream, with the +# following exceptions: +# +# 1. Add a line that describes the nvlist packing format for BEGIN records +# 2. Include DRR_OBJECT_RANGE and DRR_REDACT records in end summary +# +# The previous version of zstream does not dump opposite-endian streams +# correctly, so these don't have a comparison basis. +# + +verify_runnable "both" + +log_assert "Verify old-vs-new dump diff contains only expected additions." + +typeset sys_endian=$(get_system_endian) + +typeset -a streams=( + little-endian-long-payloads + big-endian-long-payloads + little-endian-all-drr-types-base-XDR + little-endian-all-drr-types-incr-XDR + big-endian-all-drr-types-base-XDR + big-endian-all-drr-types-incr-XDR +) + +if [[ $sys_endian == "little" ]]; then + streams+=( + little-endian-all-drr-types-base-NATIVE + little-endian-all-drr-types-incr-NATIVE + ) +else + streams+=( + big-endian-all-drr-types-base-NATIVE + big-endian-all-drr-types-incr-NATIVE + ) +fi + +typeset failed="" + +for stem in "${streams[@]}"; do + + typeset send_src="$ZSTREAM_DATADIR/${stem}.zsend.bz2" + typeset ref="$BACKDIR/${abbrev}-new.dump" + typeset out="$BACKDIR/${abbrev}-out.dump" + + typeset abbrev=$(get_stream_abbrev "$stem") + typeset send_src="$ZSTREAM_DATADIR/${stem}.zsend.bz2" + typeset old_src="$ZSTREAM_DATADIR/${abbrev}-old.dump.bz2" + typeset new_dump="$BACKDIR/${abbrev}-new.dump" + typeset old_dump="$BACKDIR/${abbrev}-old.dump" + typeset filtered="$BACKDIR/${abbrev}-filtered.dump" + + bzcat "$send_src" | zstream dump -v > "$new_dump" 2>&1 + bzcat "$old_src" > "$old_dump" + + # Remove the lines that are new additions: + # 1. "nvlist encoding = ..." lines + # 2. Summary lines for DRR_OBJECT_RANGE and DRR_REDACT + grep -v '^nvlist encoding = ' "$new_dump" | \ + grep -v 'Total DRR_OBJECT_RANGE records' | \ + grep -v 'Total DRR_REDACT records' > "$filtered" + + if ! diff -q "$old_dump" "$filtered" > /dev/null 2>&1; then + log_note "MISMATCH after filtering: $stem (abbrev $abbrev)" + log_note "$(diff "$old_dump" "$filtered")" + failed="$failed $stem" + fi +done + +[[ -z $failed ]] || \ + log_fail "Filtered new dump did not match old dump for:$failed" + +log_pass "Old-vs-new dump diff contains only expected additions." diff --git a/tests/zfs-tests/tests/functional/zstream/zstream_dump_004_neg.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_dump_004_neg.ksh new file mode 100755 index 000000000000..0bf7c82d635b --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream_dump_004_neg.ksh @@ -0,0 +1,88 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream dump on non-native-endian NATIVE-encoded streams +# eventually exits with code ENOTSUP (95 on Linux, 45 on FreeBSD) but +# still dumps the complete stream (minus nondecodable nvlists) and +# prints the rollup summary. +# +# Strategy: +# 1. Determine system endianness to identify non-native NATIVE streams +# 2. Run zstream dump -v on each non-native NATIVE stream +# 3. Verify exit code is 95 (ENOTSUP) +# 4. Verify that SUMMARY section is present and complete +# + +verify_runnable "both" + +log_assert "Non-native NATIVE-encoded streams exit with code 95." + +typeset sys_endian=$(get_system_endian) + +if [[ $sys_endian == "little" ]]; then + typeset -a streams=( + big-endian-all-drr-types-base-NATIVE + big-endian-all-drr-types-incr-NATIVE + ) +else + typeset -a streams=( + little-endian-all-drr-types-base-NATIVE + little-endian-all-drr-types-incr-NATIVE + ) +fi + +typeset failed="" + +for stem in "${streams[@]}"; do + typeset out="$BACKDIR/${stem}-out.dump" + typeset stream="$ZSTREAM_DATADIR/${stem}.zsend.bz2" + + bzcat "$stream" | zstream dump -v > "$out" 2>&1 + typeset rc=$? + + if [[ $rc -ne 45 && $rc -ne 95 ]]; then + log_note "$stem: expected exit code 45 or 95, got $rc" + failed="$failed ${stem}(rc=$rc)" + fi + + # Verify the SUMMARY section is present + if ! grep -q '^SUMMARY:' "$out"; then + log_note "$stem: missing SUMMARY section" + failed="$failed ${stem}(no-summary)" + fi + + # Verify key summary lines are present + if ! grep -q 'Total DRR_BEGIN records' "$out"; then + log_note "$stem: missing DRR_BEGIN in summary" + failed="$failed ${stem}(incomplete-summary)" + fi + + if ! grep -q 'Total stream length' "$out"; then + log_note "$stem: missing total stream length in summary" + failed="$failed ${stem}(no-stream-length)" + fi +done + +[[ -z $failed ]] || \ + log_fail "Non-native NATIVE stream check failed:$failed" + +log_pass "Non-native NATIVE-encoded streams exit with code 95." diff --git a/tests/zfs-tests/tests/functional/rsend/send-c_zstream_recompress.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_recompress_001_pos.ksh similarity index 91% rename from tests/zfs-tests/tests/functional/rsend/send-c_zstream_recompress.ksh rename to tests/zfs-tests/tests/functional/zstream/zstream_recompress_001_pos.ksh index 81c7e24a2204..ea332a32b833 100755 --- a/tests/zfs-tests/tests/functional/rsend/send-c_zstream_recompress.ksh +++ b/tests/zfs-tests/tests/functional/zstream/zstream_recompress_001_pos.ksh @@ -16,8 +16,7 @@ # Copyright (c) 2022 by Delphix. All rights reserved. # -. $STF_SUITE/tests/functional/rsend/rsend.kshlib -. $STF_SUITE/include/math.shlib +. $STF_SUITE/tests/functional/zstream/zstream.kshlib # # Description: @@ -36,10 +35,10 @@ verify_runnable "both" log_assert "Verify zstream recompress correctly modifies send streams." -log_onexit cleanup_pool $POOL2 +log_onexit cleanup_pool $POOL -typeset sendfs=$POOL2/fs -typeset recvfs=$POOL2/fs2 +typeset sendfs=$POOL/fs +typeset recvfs=$POOL/fs2 log_must zfs create -o compress=lz4 $sendfs typeset dir=$(get_prop mountpoint $sendfs) diff --git a/tests/zfs-tests/tests/functional/zstream/zstream_recompress_002_pos.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_recompress_002_pos.ksh new file mode 100755 index 000000000000..3183357d4aaf --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream_recompress_002_pos.ksh @@ -0,0 +1,51 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2024 by the OpenZFS project. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that recompressing a send stream and then decompressing it +# with zstream produces a stream identical to the original. +# +# Strategy: +# 1. Create an filesystem with compressible data +# 2. Generate a replication send stream +# 3. Pipe the stream through zstream recompress lz4 | zstream recompress off +# 4. Verify the result is byte-identical to the original stream +# + +verify_runnable "both" + +log_assert "Verify zstream recompress round-trip produces identical stream." +log_onexit cleanup_pool $POOL + +typeset sendfs=$POOL/fs + +log_must zfs create $sendfs +typeset dir=$(get_prop mountpoint $sendfs) +write_compressible $dir 16m +log_must zfs snapshot $sendfs@snap + +log_must eval "zfs send -R $sendfs@snap >$BACKDIR/original" +log_must eval "zstream recompress lz4 <$BACKDIR/original | \ + zstream recompress off >$BACKDIR/roundtrip" + +log_must cmp $BACKDIR/original $BACKDIR/roundtrip + +log_pass "zstream recompress round-trip produces identical stream." diff --git a/tests/zfs-tests/tests/functional/zstream/zstream_recompress_003_pos.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_recompress_003_pos.ksh new file mode 100755 index 000000000000..194baa43e900 --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream_recompress_003_pos.ksh @@ -0,0 +1,64 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream recompress with zstd at level 19 produces a smaller +# stream that receives with identical file contents. +# +# Strategy: +# 1. Receive the original stream and compute file hashes as baseline +# 2. Recompress the stream with zstd-19 +# 3. Verify the recompressed stream is smaller than the original +# 4. Receive the recompressed stream and verify file hashes match +# + +verify_runnable "both" + +log_assert "Verify zstream recompress with zstd-19 produces smaller stream." +log_onexit cleanup_pool $POOL + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/recompress.orig" +typeset recompressed="$BACKDIR/recompress-zstd19.out" +typeset orig_hash="$BACKDIR/hash-baseline.txt" +typeset rc_hash="$BACKDIR/hash-rc.txt" + +bzcat "$src" > "$orig" + +# Baseline: receive original and hash +recv_and_hash "$orig_hash" "$orig" cleanup + +# Recompress with zstd at level 19 +log_must eval "zstream recompress -l 19 zstd \ + < '$orig' > '$recompressed'" + +# Verify size is smaller +typeset orig_size=$(wc -c < "$orig") +typeset recomp_size=$(wc -c < "$recompressed") +log_note "Original size: $orig_size, recompressed size: $recomp_size" +[[ $recomp_size -lt $orig_size ]] || \ + log_fail "Recompressed stream ($recomp_size) not smaller than original ($orig_size)" + +# Receive recompressed and verify +recv_and_hash "$rc_hash" "$recompressed" cleanup +log_must diff "$orig_hash" "$rc_hash" + +log_pass "zstream recompress with zstd-19 produces smaller stream." diff --git a/tests/zfs-tests/tests/functional/zstream/zstream_recompress_004_pos.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_recompress_004_pos.ksh new file mode 100755 index 000000000000..c72eff14086f --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream_recompress_004_pos.ksh @@ -0,0 +1,61 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream recompress with "off" produces a larger (uncompressed) +# stream that still receives with identical file contents. +# +# Strategy: +# 1. Receive the original stream and compute file hashes as baseline +# 2. Recompress with "off" (decompress) +# 3. Verify the output stream is larger than the original +# 4. Receive and verify file hashes match +# + +verify_runnable "both" + +log_assert "Verify zstream recompress with 'off' produces larger stream." +log_onexit cleanup_pool $POOL + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/recompress.orig" +typeset uncompressed="$BACKDIR/recompress-off.out" + +bzcat "$src" > "$orig" + +# Baseline +recv_and_hash "$BACKDIR/hash-baseline.txt" "$orig" cleanup + +# Recompress with off +log_must eval "zstream recompress off < '$orig' > '$uncompressed'" + +# Verify size is larger +typeset orig_size=$(wc -c < "$orig") +typeset uncomp_size=$(wc -c < "$uncompressed") +log_note "Original size: $orig_size, uncompressed size: $uncomp_size" +[[ $uncomp_size -gt $orig_size ]] || \ + log_fail "Uncompressed stream ($uncomp_size) not larger than original ($orig_size)" + +# Receive and verify +recv_and_hash "$BACKDIR/hash-off.txt" "$uncompressed" cleanup +log_must diff "$BACKDIR/hash-baseline.txt" "$BACKDIR/hash-off.txt" + +log_pass "zstream recompress with 'off' produces larger stream." diff --git a/tests/zfs-tests/tests/functional/zstream/zstream_recompress_005_pos.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_recompress_005_pos.ksh new file mode 100755 index 000000000000..374ab2841a39 --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream_recompress_005_pos.ksh @@ -0,0 +1,62 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# Verify that zstream recompress lz4 of a zstd-5-compressed input stream +# yields a stream with a nonidentical size that zfs receives with identical +# file contents. +# +# Strategy: +# 1. Receive the original stream and compute file hashes as baseline +# 2. Recompress with lz4 +# 3. Verify the output stream size differs from the original +# 4. Receive and verify file hashes match +# + +verify_runnable "both" + +log_assert "Verify zstream recompress with lz4 preserves data." +log_onexit cleanup_pool $POOL + +typeset src="$ZSTREAM_DATADIR/decompress.zsend.bz2" +typeset orig="$BACKDIR/recompress.orig" +typeset lz4_out="$BACKDIR/recompress-lz4.out" + +bzcat "$src" > "$orig" + +# Baseline +recv_and_hash "$BACKDIR/hash-baseline.txt" "$orig" cleanup + +# Recompress with lz4 +log_must eval "zstream recompress lz4 < '$orig' > '$lz4_out'" + +# Verify size is different +typeset orig_size=$(wc -c < "$orig") +typeset lz4_size=$(wc -c < "$lz4_out") +log_note "Original size: $orig_size, lz4 size: $lz4_size" +[[ $lz4_size -ne $orig_size ]] || \ + log_fail "LZ4 stream size ($lz4_size) same as original ($orig_size)" + +# Receive and verify +recv_and_hash "$BACKDIR/hash-lz4.txt" "$lz4_out" cleanup +log_must diff "$BACKDIR/hash-baseline.txt" "$BACKDIR/hash-lz4.txt" + +log_pass "zstream recompress with lz4 preserves data." diff --git a/tests/zfs-tests/tests/functional/zstream/zstream_redup_001_pos.ksh b/tests/zfs-tests/tests/functional/zstream/zstream_redup_001_pos.ksh new file mode 100755 index 000000000000..475d4b670c38 --- /dev/null +++ b/tests/zfs-tests/tests/functional/zstream/zstream_redup_001_pos.ksh @@ -0,0 +1,79 @@ +#!/bin/ksh -p +# SPDX-License-Identifier: CDDL-1.0 + +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright (c) 2026 by Garth Snyder. All rights reserved. +# + +. $STF_SUITE/tests/functional/zstream/zstream.kshlib + +# +# Description: +# +# Verify that zstream redup produces output identical to input for same-endian +# test streams These input files contain no dedup records. However, the round +# trip does involve the full pipeline as well as validating and regenerating all +# checksums, so this is a useful check. +# +# Strategy: +# 1. For each of the same-endian test streams, decompress with bzcat +# 2. Pipe through zstream redup +# 3. Compare with cmp against the original decompressed stream +# + +verify_runnable "both" + +log_assert "Verify zstream redup is an identity transform on non-dedup streams." + +typeset sys_endian=$(get_system_endian) + +if [[ $sys_endian == "little" ]]; then + typeset -a streams=( + decompress + decompress-crypt + little-endian-all-drr-types-base-NATIVE + little-endian-all-drr-types-base-XDR + little-endian-all-drr-types-incr-NATIVE + little-endian-all-drr-types-incr-XDR + ) +else + typeset -a streams=( + big-endian-all-drr-types-base-NATIVE + big-endian-all-drr-types-base-XDR + big-endian-all-drr-types-incr-NATIVE + big-endian-all-drr-types-incr-XDR + ) +fi + +typeset failed="" + +for stem in "${streams[@]}"; do + typeset src="$ZSTREAM_DATADIR/${stem}.zsend.bz2" + typeset orig="$BACKDIR/${stem}.orig" + typeset redup_out="$BACKDIR/${stem}.redup" + + bzcat "$src" > "$orig" + zstream redup "$orig" > "$redup_out" + + if ! cmp -s "$orig" "$redup_out"; then + log_note "MISMATCH: zstream redup output differs for $stem" + failed="$failed $stem" + fi + + rm -f "$orig" "$redup_out" +done + +[[ -z $failed ]] || log_fail "Redup identity check failed for:$failed" + +log_pass "zstream redup is an identity transform on non-dedup streams."