1#!/bin/sh 2 3set -e # exit immediately on error 4# set -x # print commands before execution (debug) 5 6unset ZSTD_CLEVEL 7unset ZSTD_NBTHREADS 8 9 10die() { 11 println "$@" 1>&2 12 exit 1 13} 14 15datagen() { 16 "$DATAGEN_BIN" "$@" 17} 18 19zstd() { 20 if [ -z "$EXE_PREFIX" ]; then 21 "$ZSTD_BIN" "$@" 22 else 23 "$EXE_PREFIX" "$ZSTD_BIN" "$@" 24 fi 25} 26 27sudoZstd() { 28 if [ -z "$EXE_PREFIX" ]; then 29 sudo "$ZSTD_BIN" "$@" 30 else 31 sudo "$EXE_PREFIX" "$ZSTD_BIN" "$@" 32 fi 33} 34 35roundTripTest() { 36 if [ -n "$3" ]; then 37 cLevel="$3" 38 proba="$2" 39 else 40 cLevel="$2" 41 proba="" 42 fi 43 if [ -n "$4" ]; then 44 dLevel="$4" 45 else 46 dLevel="$cLevel" 47 fi 48 49 rm -f tmp1 tmp2 50 println "roundTripTest: datagen $1 $proba | zstd -v$cLevel | zstd -d$dLevel" 51 datagen $1 $proba | $MD5SUM > tmp1 52 datagen $1 $proba | zstd --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2 53 $DIFF -q tmp1 tmp2 54} 55 56fileRoundTripTest() { 57 if [ -n "$3" ]; then 58 local_c="$3" 59 local_p="$2" 60 else 61 local_c="$2" 62 local_p="" 63 fi 64 if [ -n "$4" ]; then 65 local_d="$4" 66 else 67 local_d="$local_c" 68 fi 69 70 rm -f tmp.zst tmp.md5.1 tmp.md5.2 71 println "fileRoundTripTest: datagen $1 $local_p > tmp && zstd -v$local_c -c tmp | zstd -d$local_d" 72 datagen $1 $local_p > tmp 73 < tmp $MD5SUM > tmp.md5.1 74 zstd --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2 75 $DIFF -q tmp.md5.1 tmp.md5.2 76} 77 78truncateLastByte() { 79 dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1" 80} 81 82println() { 83 printf '%b\n' "${*}" 84} 85 86if [ -z "${size}" ]; then 87 size= 88else 89 size=${size} 90fi 91 92SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) 93PRGDIR="$SCRIPT_DIR/../programs" 94TESTDIR="$SCRIPT_DIR/../tests" 95UNAME=${UNAME:-$(uname)} 96GREP=${GREP:-grep} 97 98case "$UNAME" in 99 SunOS) DIFF=${DIFF:-gdiff} ;; 100 *) DIFF=${DIFF:-diff} ;; 101esac 102 103detectedTerminal=false 104if [ -t 0 ] && [ -t 1 ] 105then 106 detectedTerminal=true 107fi 108isTerminal=${isTerminal:-$detectedTerminal} 109 110isWindows=false 111INTOVOID="/dev/null" 112case "$UNAME" in 113 GNU) DEVDEVICE="/dev/random" ;; 114 *) DEVDEVICE="/dev/zero" ;; 115esac 116case "$OS" in 117 Windows*) 118 isWindows=true 119 INTOVOID="NUL" 120 DEVDEVICE="NUL" 121 ;; 122esac 123 124case "$UNAME" in 125 Darwin) MD5SUM="md5 -r" ;; 126 FreeBSD) MD5SUM="gmd5sum" ;; 127 NetBSD) MD5SUM="md5 -n" ;; 128 OpenBSD) MD5SUM="md5" ;; 129 *) MD5SUM="md5sum" ;; 130esac 131 132MTIME="stat -c %Y" 133case "$UNAME" in 134 Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;; 135esac 136 137assertSameMTime() { 138 MT1=$($MTIME "$1") 139 MT2=$($MTIME "$2") 140 echo MTIME $MT1 $MT2 141 [ "$MT1" = "$MT2" ] || die "mtime on $1 doesn't match mtime on $2 ($MT1 != $MT2)" 142} 143 144GET_PERMS="stat -c %a" 145case "$UNAME" in 146 Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;; 147esac 148 149assertFilePermissions() { 150 STAT1=$($GET_PERMS "$1") 151 STAT2=$2 152 [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)" 153} 154 155assertSamePermissions() { 156 STAT1=$($GET_PERMS "$1") 157 STAT2=$($GET_PERMS "$2") 158 [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)" 159} 160 161 162# check if ZSTD_BIN is defined. if not, use the default value 163if [ -z "${ZSTD_BIN}" ]; then 164 println "\nZSTD_BIN is not set. Using the default value..." 165 ZSTD_BIN="$PRGDIR/zstd" 166fi 167 168# check if DATAGEN_BIN is defined. if not, use the default value 169if [ -z "${DATAGEN_BIN}" ]; then 170 println "\nDATAGEN_BIN is not set. Using the default value..." 171 DATAGEN_BIN="$TESTDIR/datagen" 172fi 173 174# Why was this line here ? Generates a strange ZSTD_BIN when EXE_PREFIX is non empty 175# ZSTD_BIN="$EXE_PREFIX$ZSTD_BIN" 176 177# assertions 178[ -n "$ZSTD_BIN" ] || die "zstd not found at $ZSTD_BIN! \n Please define ZSTD_BIN pointing to the zstd binary. You might also consider rebuilding zstd following the instructions in README.md" 179[ -n "$DATAGEN_BIN" ] || die "datagen not found at $DATAGEN_BIN! \n Please define DATAGEN_BIN pointing to the datagen binary. You might also consider rebuilding zstd tests following the instructions in README.md. " 180println "\nStarting playTests.sh isWindows=$isWindows EXE_PREFIX='$EXE_PREFIX' ZSTD_BIN='$ZSTD_BIN' DATAGEN_BIN='$DATAGEN_BIN'" 181 182if echo hello | zstd -v -T2 2>&1 > $INTOVOID | $GREP -q 'multi-threading is disabled' 183then 184 hasMT="" 185else 186 hasMT="true" 187fi 188 189 190zstd -vvV 191 192println "\n===> simple tests " 193 194datagen > tmp 195zstd -h 196zstd -H 197zstd -V 198println "test : basic compression " 199zstd -f tmp # trivial compression case, creates tmp.zst 200zstd -f -z tmp 201zstd -f -k tmp 202zstd -f -C tmp 203println "test : basic decompression" 204zstd -df tmp.zst # trivial decompression case (overwrites tmp) 205println "test : too large compression level => auto-fix" 206zstd -99 -f tmp # too large compression level, automatic sized down 207zstd -5000000000 -f tmp && die "too large numeric value : must fail" 208println "test : --fast aka negative compression levels" 209zstd --fast -f tmp # == -1 210zstd --fast=3 -f tmp # == -3 211zstd --fast=200000 -f tmp # too low compression level, automatic fixed 212zstd --fast=5000000000 -f tmp && die "too large numeric value : must fail" 213zstd -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0" 214println "test : too large numeric argument" 215zstd --fast=9999999999 -f tmp && die "should have refused numeric value" 216println "test : set compression level with environment variable ZSTD_CLEVEL" 217 218ZSTD_CLEVEL=12 zstd -f tmp # positive compression level 219ZSTD_CLEVEL=-12 zstd -f tmp # negative compression level 220ZSTD_CLEVEL=+12 zstd -f tmp # valid: verbose '+' sign 221ZSTD_CLEVEL='' zstd -f tmp # empty env var, warn and revert to default setting 222ZSTD_CLEVEL=- zstd -f tmp # malformed env var, warn and revert to default setting 223ZSTD_CLEVEL=a zstd -f tmp # malformed env var, warn and revert to default setting 224ZSTD_CLEVEL=+a zstd -f tmp # malformed env var, warn and revert to default setting 225ZSTD_CLEVEL=3a7 zstd -f tmp # malformed env var, warn and revert to default setting 226ZSTD_CLEVEL=50000000000 zstd -f tmp # numeric value too large, warn and revert to default setting 227println "test : override ZSTD_CLEVEL with command line option" 228ZSTD_CLEVEL=12 zstd --fast=3 -f tmp # overridden by command line option 229 230# temporary envvar changes in the above tests would actually persist in macos /bin/sh 231unset ZSTD_CLEVEL 232 233 234println "test : compress to stdout" 235zstd tmp -c > tmpCompressed 236zstd tmp --stdout > tmpCompressed # long command format 237 238println "test : compress to named file (-o)" 239rm -f tmpCompressed 240zstd tmp -o tmpCompressed 241test -f tmpCompressed # file must be created 242 243println "test : force write, correct order" 244zstd tmp -fo tmpCompressed 245 246println "test : -c + -o : last one wins" 247rm -f tmpOut 248zstd tmp -c > tmpCompressed -o tmpOut 249test -f tmpOut # file must be created 250rm -f tmpCompressed 251zstd tmp -o tmpOut -c > tmpCompressed 252test -f tmpCompressed # file must be created 253 254println "test : forgotten argument" 255cp tmp tmp2 256zstd tmp2 -fo && die "-o must be followed by filename " 257println "test : implied stdout when input is stdin" 258println bob | zstd | zstd -d 259if [ "$isTerminal" = true ]; then 260println "test : compressed data to terminal" 261println bob | zstd && die "should have refused : compressed data to terminal" 262println "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)" 263zstd -d > $INTOVOID && die "should have refused : compressed data from terminal" 264fi 265println "test : null-length file roundtrip" 266println -n '' | zstd - --stdout | zstd -d --stdout 267println "test : ensure small file doesn't add 3-bytes null block" 268datagen -g1 > tmp1 269zstd tmp1 -c | wc -c | $GREP "14" 270zstd < tmp1 | wc -c | $GREP "14" 271println "test : decompress file with wrong suffix (must fail)" 272zstd -d tmpCompressed && die "wrong suffix error not detected!" 273zstd -df tmp && die "should have refused : wrong extension" 274println "test : decompress into stdout" 275zstd -d tmpCompressed -c > tmpResult # decompression using stdout 276zstd --decompress tmpCompressed -c > tmpResult 277zstd --decompress tmpCompressed --stdout > tmpResult 278println "test : decompress from stdin into stdout" 279zstd -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout 280zstd -dc - < tmp.zst > $INTOVOID 281zstd -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used 282zstd -d - < tmp.zst > $INTOVOID 283println "test : impose memory limitation (must fail)" 284datagen -g500K > tmplimit 285zstd -f tmplimit 286zstd -d -f tmplimit.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" 287zstd -d -f tmplimit.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command 288zstd -d -f tmplimit.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command 289zstd -d -f tmplimit.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command 290rm -f tmplimit tmplimit.zst 291println "test : overwrite protection" 292zstd -q tmp && die "overwrite check failed!" 293println "test : force overwrite" 294zstd -q -f tmp 295zstd -q --force tmp 296println "test : overwrite readonly file" 297rm -f tmpro tmpro.zst 298println foo > tmpro.zst 299println foo > tmpro 300chmod 400 tmpro.zst 301zstd -q tmpro && die "should have refused to overwrite read-only file" 302zstd -q -f tmpro 303println "test: --no-progress flag" 304zstd tmpro -c --no-progress | zstd -d -f -o "$INTOVOID" --no-progress 305zstd tmpro -cv --no-progress | zstd -dv -f -o "$INTOVOID" --no-progress 306println "test: --progress flag" 307zstd tmpro -c | zstd -d -f -o "$INTOVOID" --progress 2>&1 | $GREP '[A-Za-z0-9._ ]*: [0-9]* bytes' 308zstd tmpro -c | zstd -d -f -q -o "$INTOVOID" --progress 2>&1 | $GREP '[A-Za-z0-9._ ]*: [0-9]* bytes' 309zstd tmpro -c | zstd -d -f -v -o "$INTOVOID" 2>&1 | $GREP '[A-Za-z0-9._ ]*: [0-9]* bytes' 310rm -f tmpro tmpro.zst 311println "test: overwrite input file (must fail)" 312zstd tmp -fo tmp && die "zstd compression overwrote the input file" 313zstd tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file" 314println "test: detect that input file does not exist" 315zstd nothere && die "zstd hasn't detected that input file does not exist" 316println "test: --[no-]compress-literals" 317zstd tmp -c --no-compress-literals -1 | zstd -t 318zstd tmp -c --no-compress-literals --fast=1 | zstd -t 319zstd tmp -c --no-compress-literals -19 | zstd -t 320zstd tmp -c --compress-literals -1 | zstd -t 321zstd tmp -c --compress-literals --fast=1 | zstd -t 322zstd tmp -c --compress-literals -19 | zstd -t 323zstd -b --fast=1 -i0e1 tmp --compress-literals 324zstd -b --fast=1 -i0e1 tmp --no-compress-literals 325println "test: --no-check for decompression" 326zstd -f tmp -o tmp_corrupt.zst --check 327zstd -f tmp -o tmp.zst --no-check 328printf '\xDE\xAD\xBE\xEF' | dd of=tmp_corrupt.zst bs=1 seek=$(($(wc -c < "tmp_corrupt.zst") - 4)) count=4 conv=notrunc # corrupt checksum in tmp 329zstd -d -f tmp_corrupt.zst --no-check 330zstd -d -f tmp_corrupt.zst --check --no-check # final flag overrides 331zstd -d -f tmp.zst --no-check 332 333if [ "$isWindows" = false ] && [ "$UNAME" != "AIX" ]; then 334 if [ -n "$(which readelf)" ]; then 335 println "test: check if binary has executable stack (#2963)" 336 readelf -lW "$ZSTD_BIN" | $GREP 'GNU_STACK .* RW ' || die "zstd binary has executable stack!" 337 fi 338fi 339 340println "\n===> multiple_thread test " 341 342datagen > tmp 343println "test : single-thread " 344zstd --fast --single-thread tmp -o tmpMT0 345println "test : one worker thread (default)" 346zstd --fast -T1 tmp -o tmpMT1 347println "test : two worker threads " 348zstd --fast -T2 tmp -o tmpMT2 349println "test : 16-thread " 350zstd --fast -T16 tmp -o tmpMT3 351println "test : 127-thread " 352zstd --fast -T127 tmp -o tmpMT4 353println "test : 128-thread " 354zstd --fast -T128 tmp -o tmpMT5 355println "test : max allowed numeric value is 4294967295 " 356zstd --fast -4294967295 tmp -o tmpMT6 357println "test : numeric value overflows 32-bit unsigned int " 358zstd --fast -4294967296 tmp -o tmptest9 && die "max allowed numeric value is 4294967295" 359 360datagen > tmp 361println "test : basic compression " 362zstd -f tmp # trivial compression case, creates tmp.zst 363println "test : basic decompression" 364zstd -d -f -T1 tmp.zst 365println "note : decompression does not support -T mode, but execution support" 366rm -rf tmpMT* 367 368println "\n===> --fast_argument test " 369datagen > tmp 370println "test : basic compression " 371zstd -f tmp # trivial compression case, creates tmp.zst 372println "test: --fast=1" 373zstd --fast=1 -f tmp 374println "test: --fast=99" 375zstd --fast=99 -f tmp 376println "test: Invalid value -- negative number" 377zstd --fast=-1 -f tmp && die "error: Invalid value -- negative number" 378println "test: Invalid value -- zero" 379zstd --fast=0 -f tmp && die "error: Invalid value -- 0 number" 380println "test: max allowed numeric argument of --fast is 4294967295" 381zstd --fast=4294967295 -f tmp 382println "test: numeric value overflows 32-bit unsigned int " 383zstd --fast=4294967296 -f tmp && die "max allowed argument of --fast is 4294967295" 384 385println "\n===> --exclude-compressed flag" 386rm -rf precompressedFilterTestDir 387mkdir -p precompressedFilterTestDir 388datagen $size > precompressedFilterTestDir/input.5 389datagen $size > precompressedFilterTestDir/input.6 390zstd --exclude-compressed --long --rm -r precompressedFilterTestDir 391datagen $size > precompressedFilterTestDir/input.7 392datagen $size > precompressedFilterTestDir/input.8 393zstd --exclude-compressed --long --rm -r precompressedFilterTestDir 394test ! -f precompressedFilterTestDir/input.5.zst.zst 395test ! -f precompressedFilterTestDir/input.6.zst.zst 396file1timestamp=`$MTIME precompressedFilterTestDir/input.5.zst` 397file2timestamp=`$MTIME precompressedFilterTestDir/input.7.zst` 398if [ $file2timestamp -ge $file1timestamp ]; then 399 println "Test is successful. input.5.zst is precompressed and therefore not compressed/modified again." 400else 401 println "Test is not successful" 402fi 403# File Extension check. 404datagen $size > precompressedFilterTestDir/input.zstbar 405zstd --exclude-compressed --long --rm -r precompressedFilterTestDir 406# zstd should compress input.zstbar 407test -f precompressedFilterTestDir/input.zstbar.zst 408# Check without the --exclude-compressed flag 409zstd --long --rm -r precompressedFilterTestDir 410# Files should get compressed again without the --exclude-compressed flag. 411test -f precompressedFilterTestDir/input.5.zst.zst 412test -f precompressedFilterTestDir/input.6.zst.zst 413 414# Test some other compressed file extensions 415datagen $size > precompressedFilterTestDir/input.flac 416datagen $size > precompressedFilterTestDir/input.mov 417datagen $size > precompressedFilterTestDir/input.mp3 418zstd --exclude-compressed --long --rm -r precompressedFilterTestDir 419test ! -f precompressedFilterTestDir/input.flac.zst 420test ! -f precompressedFilterTestDir/input.mov.zst 421test ! -f precompressedFilterTestDir/input.mp3.zst 422zstd --long --rm -r precompressedFilterTestDir 423test -f precompressedFilterTestDir/input.flac.zst 424test -f precompressedFilterTestDir/input.mov.zst 425test -f precompressedFilterTestDir/input.mp3.zst 426rm -rf precompressedFilterTestDir 427println "Test completed" 428 429 430 431println "\n===> warning prompts should not occur if stdin is an input" 432println "y" > tmpPrompt 433println "hello world" >> tmpPrompt 434zstd tmpPrompt -f 435zstd < tmpPrompt -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" 436zstd < tmpPrompt -o tmpPrompt.zst -f # should successfully overwrite with -f 437zstd -q -d -f tmpPrompt.zst -o tmpPromptRegenerated 438$DIFF tmpPromptRegenerated tmpPrompt # the first 'y' character should not be swallowed 439 440echo 'yes' | zstd tmpPrompt -v -o tmpPrompt.zst # accept piped "y" input to force overwrite when using files 441echo 'yes' | zstd < tmpPrompt -v -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" 442zstd tmpPrompt - < tmpPrompt -o tmpPromp.zst --rm && die "should have aborted immediately and failed to remove" 443 444println "Test completed" 445 446 447println "\n===> recursive mode test " 448# combination of -r with empty list of input file 449zstd -c -r < tmp > tmp.zst 450 451# combination of -r with empty folder 452mkdir -p tmpEmptyDir 453zstd -r tmpEmptyDir 454rm -rf tmpEmptyDir 455 456 457println "\n===> file removal" 458zstd -f --rm tmp 459test ! -f tmp # tmp should no longer be present 460zstd -f -d --rm tmp.zst 461test ! -f tmp.zst # tmp.zst should no longer be present 462println "test: --rm is disabled when output is stdout" 463test -f tmp 464zstd --rm tmp -c > $INTOVOID 465test -f tmp # tmp shall still be there 466zstd --rm tmp --stdout > $INTOVOID 467test -f tmp # tmp shall still be there 468zstd -f --rm tmp -c > $INTOVOID 469test -f tmp # tmp shall still be there 470zstd -f tmp -c > $INTOVOID --rm 471test -f tmp # tmp shall still be there 472println "test: --rm is disabled when multiple inputs are concatenated into a single output" 473cp tmp tmp2 474zstd --rm tmp tmp2 -c > $INTOVOID 475test -f tmp 476test -f tmp2 477rm -f tmp3.zst 478echo 'y' | zstd -v tmp tmp2 -o tmp3.zst --rm # prompt for confirmation 479test -f tmp 480test -f tmp2 481zstd -f tmp tmp2 -o tmp3.zst --rm # just warns, no prompt 482test -f tmp 483test -f tmp2 484zstd -q tmp tmp2 -o tmp3.zst --rm && die "should refuse to concatenate" 485println "test: --rm is active with -o when single input" 486rm -f tmp2.zst 487zstd --rm tmp2 -o tmp2.zst 488test -f tmp2.zst 489test ! -f tmp2 490println "test: -c followed by -o => -o wins, so --rm remains active" # (#3719) 491rm tmp2.zst 492cp tmp tmp2 493zstd --rm tmp2 -c > $INTOVOID -o tmp2.zst 494test ! -f tmp2 495println "test: -o followed by -c => -c wins, so --rm is disabled" # (#3719) 496rm tmp3.zst 497cp tmp tmp2 498zstd -v --rm tmp2 -o tmp2.zst -c > tmp3.zst 499test -f tmp2 500test -f tmp3.zst 501println "test : should quietly not remove non-regular file" 502println hello > tmp 503zstd tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID" 504$GREP "Refusing to remove non-regular file" tmplog && die 505rm -f tmplog 506zstd tmp -f -o "$INTOVOID" 2>&1 | $GREP "Refusing to remove non-regular file" && die 507println "test : --rm on stdin" 508println a | zstd --rm > $INTOVOID # --rm should remain silent 509rm -f tmp 510zstd -f tmp && die "tmp not present : should have failed" 511test ! -f tmp.zst # tmp.zst should not be created 512println "test : -d -f do not delete destination when source is not present" 513touch tmp # create destination file 514zstd -d -f tmp.zst && die "attempt to decompress a non existing file" 515test -f tmp # destination file should still be present 516println "test : -f do not delete destination when source is not present" 517rm -f tmp # erase source file 518touch tmp.zst # create destination file 519zstd -f tmp && die "attempt to compress a non existing file" 520test -f tmp.zst # destination file should still be present 521rm -rf tmp* # may also erase tmp* directory from previous failed run 522 523 524println "\n===> decompression only tests " 525# the following test verifies that the decoder is compatible with RLE as first block 526# older versions of zstd cli are not able to decode such corner case. 527# As a consequence, the zstd cli do not generate them, to maintain compatibility with older versions. 528dd bs=1048576 count=1 if=/dev/zero of=tmp 529zstd -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst" 530$DIFF -s tmp1 tmp 531 532touch tmp_empty 533zstd -d -o tmp2 "$TESTDIR/golden-decompression/empty-block.zst" 534$DIFF -s tmp2 tmp_empty 535 536zstd -t "$TESTDIR/golden-decompression/zeroSeq_2B.zst" 537 538zstd -t "$TESTDIR/golden-decompression-errors/zeroSeq_extraneous.zst" && die "invalid Sequences section should have been detected" 539 540rm -f tmp* 541 542println "\n===> compress multiple files" 543println hello > tmp1 544println world > tmp2 545zstd tmp1 tmp2 -o "$INTOVOID" -f 546zstd tmp1 tmp2 -c | zstd -t 547echo 'y' | zstd -v tmp1 tmp2 -o tmp.zst 548test ! -f tmp1.zst 549test ! -f tmp2.zst 550zstd tmp1 tmp2 551zstd -t tmp1.zst tmp2.zst 552zstd -dc tmp1.zst tmp2.zst 553zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f 554echo 'y' | zstd -v -d tmp1.zst tmp2.zst -o tmp 555touch tmpexists 556zstd tmp1 tmp2 -f -o tmpexists 557zstd tmp1 tmp2 -q -o tmpexists && die "should have refused to overwrite" 558println gooder > tmp_rm1 559println boi > tmp_rm2 560println worldly > tmp_rm3 561echo 'y' | zstd -v tmp_rm1 tmp_rm2 -v -o tmp_rm3.zst 562test -f tmp_rm1 563test -f tmp_rm2 564cp tmp_rm3.zst tmp_rm4.zst 565echo 'Y' | zstd -v -d tmp_rm3.zst tmp_rm4.zst -v -o tmp_rm_out --rm 566test -f tmp_rm3.zst 567test -f tmp_rm4.zst 568println gooder > tmpexists1 569zstd tmpexists1 tmpexists -c --rm -f > $INTOVOID 570# Bug: PR #972 571if [ "$?" -eq 139 ]; then 572 die "should not have segfaulted" 573fi 574test -f tmpexists1 575test -f tmpexists 576println "\n===> multiple files and shell completion " 577datagen -s1 > tmp1 2> $INTOVOID 578datagen -s2 -g100K > tmp2 2> $INTOVOID 579datagen -s3 -g1M > tmp3 2> $INTOVOID 580println "compress tmp* : " 581zstd -f tmp* 582test -f tmp1.zst 583test -f tmp2.zst 584test -f tmp3.zst 585rm -f tmp1 tmp2 tmp3 586println "decompress tmp* : " 587zstd -df ./*.zst 588test -f tmp1 589test -f tmp2 590test -f tmp3 591println "compress tmp* into stdout > tmpall : " 592zstd -c tmp1 tmp2 tmp3 > tmpall 593test -f tmpall # should check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst) 594println "decompress tmpall* into stdout > tmpdec : " 595cp tmpall tmpall2 596zstd -dc tmpall* > tmpdec 597test -f tmpdec # should check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3)) 598println "compress multiple files including a missing one (notHere) : " 599zstd -f tmp1 notHere tmp2 && die "missing file not detected!" 600rm -f tmp* 601 602 603if [ "$isWindows" = false ] ; then 604 println "\n===> zstd fifo named pipe test " 605 echo "Hello World!" > tmp_original 606 mkfifo tmp_named_pipe 607 # note : fifo test doesn't work in combination with `dd` or `cat` 608 echo "Hello World!" > tmp_named_pipe & 609 zstd tmp_named_pipe -o tmp_compressed 610 zstd -d -o tmp_decompressed tmp_compressed 611 $DIFF -s tmp_original tmp_decompressed 612 rm -rf tmp* 613fi 614 615println "\n===> zstd created file permissions tests" 616if [ "$isWindows" = false ] ; then 617 rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out # todo: remove 618 619 ORIGINAL_UMASK=$(umask) 620 umask 0000 621 622 datagen > tmp1 623 datagen > tmp2 624 assertFilePermissions tmp1 666 625 assertFilePermissions tmp2 666 626 627 println "test : copy 666 permissions in file -> file compression " 628 zstd -f tmp1 -o tmp1.zst 629 assertSamePermissions tmp1 tmp1.zst 630 println "test : copy 666 permissions in file -> file decompression " 631 zstd -f -d tmp1.zst -o tmp1.out 632 assertSamePermissions tmp1.zst tmp1.out 633 634 rm -f tmp1.zst tmp1.out 635 636 println "test : copy 400 permissions in file -> file compression (write to a read-only file) " 637 chmod 0400 tmp1 638 assertFilePermissions tmp1 400 639 zstd -f tmp1 -o tmp1.zst 640 assertSamePermissions tmp1 tmp1.zst 641 println "test : copy 400 permissions in file -> file decompression (write to a read-only file) " 642 zstd -f -d tmp1.zst -o tmp1 643 assertSamePermissions tmp1.zst tmp1 644 645 rm -f tmp1.zst tmp1.out 646 647 println "test : check created permissions from stdin input in compression " 648 zstd -f -o tmp1.zst < tmp1 649 assertFilePermissions tmp1.zst 666 650 println "test : check created permissions from stdin input in decompression " 651 zstd -f -d -o tmp1.out < tmp1.zst 652 assertFilePermissions tmp1.out 666 653 654 rm -f tmp1.zst tmp1.out 655 656 println "test : check created permissions from multiple inputs in compression " 657 zstd -f tmp1 tmp2 -o tmp1.zst 658 assertFilePermissions tmp1.zst 666 659 println "test : check created permissions from multiple inputs in decompression " 660 cp tmp1.zst tmp2.zst 661 zstd -f -d tmp1.zst tmp2.zst -o tmp1.out 662 assertFilePermissions tmp1.out 666 663 664 rm -f tmp1.zst tmp2.zst tmp1.out tmp2.out 665 666 println "test : check permissions on pre-existing output file in compression " 667 chmod 0600 tmp1 668 touch tmp1.zst 669 chmod 0400 tmp1.zst 670 zstd -f tmp1 -o tmp1.zst 671 assertFilePermissions tmp1.zst 600 672 println "test : check permissions on pre-existing output file in decompression " 673 chmod 0400 tmp1.zst 674 touch tmp1.out 675 chmod 0200 tmp1.out 676 zstd -f -d tmp1.zst -o tmp1.out 677 assertFilePermissions tmp1.out 400 678 679 umask 0666 680 chmod 0666 tmp1 tmp2 681 682 rm -f tmp1.zst tmp1.out 683 684 println "test : respect umask when compressing from stdin input " 685 zstd -f -o tmp1.zst < tmp1 686 assertFilePermissions tmp1.zst 0 687 println "test : respect umask when decompressing from stdin input " 688 chmod 0666 tmp1.zst 689 zstd -f -d -o tmp1.out < tmp1.zst 690 assertFilePermissions tmp1.out 0 691 692 rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out 693 umask $ORIGINAL_UMASK 694fi 695 696if [ -n "$DEVNULLRIGHTS" ] ; then 697 # these tests requires sudo rights, which is uncommon. 698 # they are only triggered if DEVNULLRIGHTS macro is defined. 699 println "\n===> checking /dev/null permissions are unaltered " 700 datagen > tmp 701 sudoZstd tmp -o $INTOVOID # sudo rights could modify /dev/null permissions 702 sudoZstd tmp -c > $INTOVOID 703 zstd tmp -f -o tmp.zst 704 sudoZstd -d tmp.zst -c > $INTOVOID 705 sudoZstd -d tmp.zst -o $INTOVOID 706 ls -las $INTOVOID | $GREP "rw-rw-rw-" 707fi 708 709if [ -n "$READFROMBLOCKDEVICE" ] ; then 710 # This creates a temporary block device, which is only possible on unix-y 711 # systems, is somewhat invasive, and requires sudo. For these reasons, you 712 # have to specifically ask for this test. 713 println "\n===> checking that zstd can read from a block device" 714 datagen -g65536 > tmp.img 715 sudo losetup -fP tmp.img 716 LOOP_DEV=$(losetup -a | $GREP 'tmp\.img' | cut -f1 -d:) 717 [ -z "$LOOP_DEV" ] && die "failed to get loopback device" 718 sudoZstd $LOOP_DEV -c > tmp.img.zst && die "should fail without -f" 719 sudoZstd -f $LOOP_DEV -c > tmp.img.zst 720 zstd -d tmp.img.zst -o tmp.img.copy 721 sudo losetup -d $LOOP_DEV 722 $DIFF -s tmp.img tmp.img.copy || die "round trip failed" 723 rm -f tmp.img tmp.img.zst tmp.img.copy 724fi 725 726println "\n===> zstd created file timestamp tests" 727datagen > tmp 728touch -m -t 200001010000.00 tmp 729println "test : copy mtime in file -> file compression " 730zstd -f tmp -o tmp.zst 731assertSameMTime tmp tmp.zst 732println "test : copy mtime in file -> file decompression " 733zstd -f -d tmp.zst -o tmp.out 734assertSameMTime tmp.zst tmp.out 735rm -f tmp 736 737println "\n===> compress multiple files into an output directory, --output-dir-flat" 738println henlo > tmp1 739mkdir tmpInputTestDir 740mkdir tmpInputTestDir/we 741mkdir tmpInputTestDir/we/must 742mkdir tmpInputTestDir/we/must/go 743mkdir tmpInputTestDir/we/must/go/deeper 744println cool > tmpInputTestDir/we/must/go/deeper/tmp2 745mkdir tmpOutDir 746zstd tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir 747test -f tmpOutDir/tmp1.zst 748test -f tmpOutDir/tmp2.zst 749println "test : decompress multiple files into an output directory, --output-dir-flat" 750mkdir tmpOutDirDecomp 751zstd tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp 752test -f tmpOutDirDecomp/tmp2 753test -f tmpOutDirDecomp/tmp1 754rm -f tmpOutDirDecomp/* 755zstd tmpOutDir -r -d --output-dir-flat=tmpOutDirDecomp 756test -f tmpOutDirDecomp/tmp2 757test -f tmpOutDirDecomp/tmp1 758rm -rf tmp* 759 760if [ "$isWindows" = false ] ; then 761 println "\n===> compress multiple files into an output directory and mirror input folder, --output-dir-mirror" 762 println "test --output-dir-mirror" > tmp1 763 mkdir -p tmpInputTestDir/we/.../..must/go/deeper.. 764 println cool > tmpInputTestDir/we/.../..must/go/deeper../tmp2 765 zstd tmp1 -r tmpInputTestDir --output-dir-mirror tmpOutDir 766 test -f tmpOutDir/tmp1.zst 767 test -f tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2.zst 768 769 println "test: compress input dir will be ignored if it has '..'" 770 zstd -r tmpInputTestDir/we/.../..must/../..mustgo/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'" 771 zstd -r tmpInputTestDir/we/.../..must/deeper../.. --output-dir-mirror non-exist && die "input cannot contain '..'" 772 zstd -r ../tests/tmpInputTestDir/we/.../..must/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'" 773 test ! -d non-exist 774 775 println "test: compress input dir should succeed with benign uses of '..'" 776 zstd -r tmpInputTestDir/we/.../..must/go/deeper.. --output-dir-mirror tmpout 777 test -d tmpout 778 779 println "test : decompress multiple files into an output directory, --output-dir-mirror" 780 zstd tmpOutDir -r -d --output-dir-mirror tmpOutDirDecomp 781 test -f tmpOutDirDecomp/tmpOutDir/tmp1 782 test -f tmpOutDirDecomp/tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2 783 784 println "test: decompress input dir will be ignored if it has '..'" 785 zstd -r tmpOutDir/tmpInputTestDir/we/.../..must/../..must --output-dir-mirror non-exist && die "input cannot contain '..'" 786 test ! -d non-exist 787 788 rm -rf tmp* 789fi 790 791 792println "test : compress multiple files reading them from a file, --filelist=FILE" 793println "Hello world!, file1" > tmp1 794println "Hello world!, file2" > tmp2 795println tmp1 > tmp_fileList 796println tmp2 >> tmp_fileList 797zstd -f --filelist=tmp_fileList 798test -f tmp2.zst 799test -f tmp1.zst 800 801println "test : alternate syntax: --filelist FILE" 802zstd -f --filelist tmp_fileList 803test -f tmp2.zst 804test -f tmp1.zst 805 806println "test : reading file list from a symlink, --filelist=FILE" 807rm -f *.zst 808ln -s tmp_fileList tmp_symLink 809zstd -f --filelist=tmp_symLink 810test -f tmp2.zst 811test -f tmp1.zst 812 813println "test : compress multiple files reading them from multiple files, --filelist=FILE" 814rm -f *.zst 815println "Hello world!, file3" > tmp3 816println "Hello world!, file4" > tmp4 817println tmp3 > tmp_fileList2 818println tmp4 >> tmp_fileList2 819zstd -f --filelist=tmp_fileList --filelist=tmp_fileList2 820test -f tmp1.zst 821test -f tmp2.zst 822test -f tmp3.zst 823test -f tmp4.zst 824 825println "test : decompress multiple files reading them from a file, --filelist=FILE" 826rm -f tmp1 tmp2 827println tmp1.zst > tmpZst 828println tmp2.zst >> tmpZst 829zstd -d -f --filelist=tmpZst 830test -f tmp1 831test -f tmp2 832 833println "test : decompress multiple files reading them from multiple files, --filelist=FILE" 834rm -f tmp1 tmp2 tmp3 tmp4 835println tmp3.zst > tmpZst2 836println tmp4.zst >> tmpZst2 837zstd -d -f --filelist=tmpZst --filelist=tmpZst2 838test -f tmp1 839test -f tmp2 840test -f tmp3 841test -f tmp4 842 843println "test : survive the list of files with too long filenames (--filelist=FILE)" 844datagen -g5M > tmp_badList 845zstd -qq -f --filelist=tmp_badList && die "should have failed : file name length is too long" # printing very long text garbage on console will cause CI failure 846 847println "test : survive a list of files which is text garbage (--filelist=FILE)" 848datagen > tmp_badList 849zstd -qq -f --filelist=tmp_badList && die "should have failed : list is text garbage" # printing very long text garbage on console will cause CI failure 850 851println "test : survive a list of files which is binary garbage (--filelist=FILE)" 852datagen -P0 -g1M > tmp_badList 853zstd -qq -f --filelist=tmp_badList && die "should have failed : list is binary garbage" # let's avoid printing binary garbage on console 854 855println "test : try to overflow internal list of files (--filelist=FILE)" 856touch tmp1 tmp2 tmp3 tmp4 tmp5 tmp6 857ls tmp* > tmpList 858zstd -f tmp1 --filelist=tmpList --filelist=tmpList tmp2 tmp3 # can trigger an overflow of internal file list 859rm -rf tmp* 860 861println "\n===> --[no-]content-size tests" 862 863datagen > tmp_contentsize 864zstd -f tmp_contentsize 865zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" 866zstd -f --no-content-size tmp_contentsize 867zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" && die 868zstd -f --content-size tmp_contentsize 869zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" 870zstd -f --content-size --no-content-size tmp_contentsize 871zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" && die 872rm -rf tmp* 873 874println "test : show-default-cparams regular" 875datagen > tmp 876zstd --show-default-cparams -f tmp 877zstd --show-default-cparams -d tmp.zst && die "error: can't use --show-default-cparams in decompression mode" 878rm -rf tmp* 879 880println "test : show-default-cparams recursive" 881mkdir tmp_files 882datagen -g15000 > tmp_files/tmp1 883datagen -g129000 > tmp_files/tmp2 884datagen -g257000 > tmp_files/tmp3 885zstd --show-default-cparams -f -r tmp_files 886rm -rf tmp* 887 888println "test : show compression parameters in verbose mode" 889datagen > tmp 890zstd -vv tmp 2>&1 | \ 891$GREP -q -- "--zstd=wlog=[0-9]*,clog=[0-9]*,hlog=[0-9]*,slog=[0-9]*,mml=[0-9]*,tlen=[0-9]*,strat=[0-9]*" 892rm -rf tmp* 893 894println "\n===> Advanced compression parameters " 895println "Hello world!" | zstd --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" 896println "Hello world!" | zstd --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" 897println "Hello world!" | zstd --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!" 898println "Hello world!" | zstd --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound not detected!" # > btultra2 : does not exist 899test ! -f tmp.zst # tmp.zst should not be created 900roundTripTest -g512K 901roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6" 902roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" 903roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6" 904roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7" 905roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7" 906roundTripTest -g64K "19 --zstd=strat=9" # btultra2 907 908 909println "\n===> Pass-Through mode " 910println "Hello world 1!" | zstd -df 911println "Hello world 2!" | zstd -dcf 912println "Hello world 3!" > tmp1 913zstd -dcf tmp1 914println "" | zstd -df > tmp1 915println "" > tmp2 916$DIFF -q tmp1 tmp2 917println "1" | zstd -df > tmp1 918println "1" > tmp2 919$DIFF -q tmp1 tmp2 920println "12" | zstd -df > tmp1 921println "12" > tmp2 922$DIFF -q tmp1 tmp2 923rm -rf tmp* 924 925 926println "\n===> frame concatenation " 927println "hello " > hello.tmp 928println "world!" > world.tmp 929cat hello.tmp world.tmp > helloworld.tmp 930zstd -c hello.tmp > hello.zst 931zstd -c world.tmp > world.zst 932zstd -c hello.tmp world.tmp > helloworld.zst 933zstd -dc helloworld.zst > result.tmp 934$DIFF helloworld.tmp result.tmp 935cat hello.zst world.zst > helloworld.zst 936zstd -dc helloworld.zst > result.tmp 937cat result.tmp 938$DIFF helloworld.tmp result.tmp 939println "frame concatenation without checksum" 940zstd -c hello.tmp > hello.zst --no-check 941zstd -c world.tmp > world.zst --no-check 942cat hello.zst world.zst > helloworld.zstd 943zstd -dc helloworld.zst > result.tmp 944$DIFF helloworld.tmp result.tmp 945println "testing zstdcat symlink" 946ln -sf "$ZSTD_BIN" zstdcat 947$EXE_PREFIX ./zstdcat helloworld.zst > result.tmp 948$DIFF helloworld.tmp result.tmp 949ln -s helloworld.zst helloworld.link.zst 950$EXE_PREFIX ./zstdcat helloworld.link.zst > result.tmp 951$DIFF helloworld.tmp result.tmp 952rm -f zstdcat 953rm -f result.tmp 954println "testing zcat symlink" 955ln -sf "$ZSTD_BIN" zcat 956$EXE_PREFIX ./zcat helloworld.zst > result.tmp 957$DIFF helloworld.tmp result.tmp 958$EXE_PREFIX ./zcat helloworld.link.zst > result.tmp 959$DIFF helloworld.tmp result.tmp 960rm -f zcat 961rm -f ./*.tmp ./*.zstd 962println "frame concatenation tests completed" 963 964 965if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] && [ "$UNAME" != "OpenBSD" ] && [ "$UNAME" != "AIX" ]; then 966println "\n**** flush write error test **** " 967 968println "println foo | zstd > /dev/full" 969println foo | zstd > /dev/full && die "write error not detected!" 970println "println foo | zstd | zstd -d > /dev/full" 971println foo | zstd | zstd -d > /dev/full && die "write error not detected!" 972 973fi 974 975 976if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then 977 978println "\n===> symbolic link test " 979 980rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst 981println "hello world" > hello.tmp 982ln -s hello.tmp world.tmp 983ln -s hello.tmp world2.tmp 984zstd world.tmp hello.tmp || true 985test -f hello.tmp.zst # regular file should have been compressed! 986test ! -f world.tmp.zst # symbolic link should not have been compressed! 987zstd world.tmp || true 988test ! -f world.tmp.zst # symbolic link should not have been compressed! 989zstd world.tmp world2.tmp || true 990test ! -f world.tmp.zst # symbolic link should not have been compressed! 991test ! -f world2.tmp.zst # symbolic link should not have been compressed! 992zstd world.tmp hello.tmp -f 993test -f world.tmp.zst # symbolic link should have been compressed with --force 994rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst 995 996fi 997 998 999println "\n===> test sparse file support " 1000 1001datagen -g5M -P100 > tmpSparse 1002zstd tmpSparse -c | zstd -dv -o tmpSparseRegen 1003$DIFF -s tmpSparse tmpSparseRegen 1004zstd tmpSparse -c | zstd -dv --sparse -c > tmpOutSparse 1005$DIFF -s tmpSparse tmpOutSparse 1006zstd tmpSparse -c | zstd -dv --no-sparse -c > tmpOutNoSparse 1007$DIFF -s tmpSparse tmpOutNoSparse 1008ls -ls tmpSparse* # look at file size and block size on disk 1009datagen -s1 -g1200007 -P100 | zstd | zstd -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) 1010datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd 1011ls -ls tmpSparseOdd # look at file size and block size on disk 1012println "\n Sparse Compatibility with Console :" 1013println "Hello World 1 !" | zstd | zstd -d -c 1014println "Hello World 2 !" | zstd | zstd -d | cat 1015println "\n Sparse Compatibility with Append :" 1016datagen -P100 -g1M > tmpSparse1M 1017cat tmpSparse1M tmpSparse1M > tmpSparse2M 1018zstd -v -f tmpSparse1M -o tmpSparseCompressed 1019zstd -d -v -f tmpSparseCompressed -o tmpSparseRegenerated 1020zstd -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated 1021ls -ls tmpSparse* # look at file size and block size on disk 1022$DIFF tmpSparse2M tmpSparseRegenerated 1023rm -f tmpSparse* 1024 1025 1026println "\n===> stream-size mode" 1027 1028datagen -g11000 > tmp 1029println "test : basic file compression vs sized streaming compression" 1030file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) 1031stream_size=$(cat tmp | zstd -14 --stream-size=11000 | wc -c) 1032if [ "$stream_size" -gt "$file_size" ]; then 1033 die "hinted compression larger than expected" 1034fi 1035println "test : sized streaming compression and decompression" 1036cat tmp | zstd -14 -f tmp -o tmp.zst --stream-size=11000 1037zstd -df tmp.zst -o tmp_decompress 1038cmp tmp tmp_decompress || die "difference between original and decompressed file" 1039println "test : incorrect stream size" 1040cat tmp | zstd -14 -f -o tmp.zst --stream-size=11001 && die "should fail with incorrect stream size" 1041 1042println "\n===> zstd zero weight dict test " 1043rm -f tmp* 1044cp "$TESTDIR/dict-files/zero-weight-dict" tmp_input 1045zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input 1046zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp 1047$DIFF tmp_decomp tmp_input 1048rm -rf tmp* 1049 1050println "\n===> zstd (valid) zero weight dict test " 1051rm -f tmp* 1052# 0 has a non-zero weight in the dictionary 1053echo "0000000000000000000000000" > tmp_input 1054zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input 1055zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp 1056$DIFF tmp_decomp tmp_input 1057rm -rf tmp* 1058 1059println "\n===> size-hint mode" 1060 1061datagen -g11000 > tmp 1062datagen -g11000 > tmp2 1063datagen > tmpDict 1064println "test : basic file compression vs hinted streaming compression" 1065file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) 1066stream_size=$(cat tmp | zstd -14 --size-hint=11000 | wc -c) 1067if [ "$stream_size" -ge "$file_size" ]; then 1068 die "hinted compression larger than expected" 1069fi 1070println "test : hinted streaming compression and decompression" 1071cat tmp | zstd -14 -f -o tmp.zst --size-hint=11000 1072zstd -df tmp.zst -o tmp_decompress 1073cmp tmp tmp_decompress || die "difference between original and decompressed file" 1074println "test : hinted streaming compression with dictionary" 1075cat tmp | zstd -14 -f -D tmpDict --size-hint=11000 | zstd -t -D tmpDict 1076println "test : multiple file compression with hints and dictionary" 1077zstd -14 -f -D tmpDict --size-hint=11000 tmp tmp2 1078zstd -14 -f -o tmp1_.zst -D tmpDict --size-hint=11000 tmp 1079zstd -14 -f -o tmp2_.zst -D tmpDict --size-hint=11000 tmp2 1080cmp tmp.zst tmp1_.zst || die "first file's output differs" 1081cmp tmp2.zst tmp2_.zst || die "second file's output differs" 1082println "test : incorrect hinted stream sizes" 1083cat tmp | zstd -14 -f --size-hint=11050 | zstd -t # slightly too high 1084cat tmp | zstd -14 -f --size-hint=10950 | zstd -t # slightly too low 1085cat tmp | zstd -14 -f --size-hint=22000 | zstd -t # considerably too high 1086cat tmp | zstd -14 -f --size-hint=5500 | zstd -t # considerably too low 1087println "test : allows and interprets K,KB,KiB,M,MB and MiB suffix" 1088cat tmp | zstd -14 -f --size-hint=11K | zstd -t 1089cat tmp | zstd -14 -f --size-hint=11KB | zstd -t 1090cat tmp | zstd -14 -f --size-hint=11KiB | zstd -t 1091cat tmp | zstd -14 -f --size-hint=1M | zstd -t 1092cat tmp | zstd -14 -f --size-hint=1MB | zstd -t 1093cat tmp | zstd -14 -f --size-hint=1MiB | zstd -t 1094 1095 1096println "\n===> dictionary tests " 1097println "- Test high/low compressibility corpus training" 1098datagen -g12M -P90 > tmpCorpusHighCompress 1099datagen -g12M -P5 > tmpCorpusLowCompress 1100zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress 1101zstd --train -B2K tmpCorpusLowCompress -o tmpDictLowCompress 1102rm -f tmpCorpusHighCompress tmpCorpusLowCompress tmpDictHighCompress tmpDictLowCompress 1103println "- Test with raw dict (content only) " 1104datagen > tmpDict 1105datagen -g1M | $MD5SUM > tmp1 1106datagen -g1M | zstd -D tmpDict | zstd -D tmpDict -dvq | $MD5SUM > tmp2 1107$DIFF -q tmp1 tmp2 1108println "- Create first dictionary " 1109TESTFILE="$PRGDIR"/zstdcli.c 1110zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 1111cp "$TESTFILE" tmp 1112println "- Test dictionary compression with tmpDict as an input file and dictionary" 1113zstd -f tmpDict -D tmpDict && die "compression error not detected!" 1114println "- Dictionary compression roundtrip" 1115zstd -f tmp -D tmpDict 1116zstd -d tmp.zst -D tmpDict -fo result 1117$DIFF "$TESTFILE" result 1118println "- Dictionary compression with hlog < clog" 1119zstd -6f tmp -D tmpDict --zstd=clog=25,hlog=23 1120println "- Dictionary compression with btlazy2 strategy" 1121zstd -f tmp -D tmpDict --zstd=strategy=6 1122zstd -d tmp.zst -D tmpDict -fo result 1123$DIFF "$TESTFILE" result 1124if [ -e /proc/self/fd/0 ]; then 1125 println "- Test rejecting irregular dictionary file" 1126 cat tmpDict | zstd -f tmp -D /proc/self/fd/0 && die "Piped dictionary should fail!" 1127 cat tmpDict | zstd -d tmp.zst -D /proc/self/fd/0 -f && die "Piped dictionary should fail!" 1128fi 1129if [ -n "$hasMT" ] 1130then 1131 println "- Test dictionary compression with multithreading " 1132 datagen -g5M | zstd -T2 -D tmpDict | zstd -t -D tmpDict # fails with v1.3.2 1133fi 1134println "- Create second (different) dictionary " 1135zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 1136zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 1137println "- Create dictionary with short dictID" 1138zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 1139cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 1140println "- Create dictionary with wrong dictID parameter order (must fail)" 1141zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument " 1142println "- Create dictionary with size limit" 1143zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K -v 1144println "- Create dictionary with small size limit" 1145zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict=1K -v 1146println "- Create dictionary with wrong parameter order (must fail)" 1147zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument " 1148println "- Compress without dictID" 1149zstd -f tmp -D tmpDict1 --no-dictID 1150zstd -d tmp.zst -D tmpDict -fo result 1151$DIFF "$TESTFILE" result 1152println "- Compress multiple files with dictionary" 1153rm -rf dirTestDict 1154mkdir dirTestDict 1155cp "$TESTDIR"/*.c dirTestDict 1156cp "$PRGDIR"/*.c dirTestDict 1157cp "$PRGDIR"/*.h dirTestDict 1158$MD5SUM dirTestDict/* > tmph1 1159zstd -f --rm dirTestDict/* -D tmpDictC 1160zstd -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default 1161case "$UNAME" in 1162 Darwin) println "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5 1163 *) $MD5SUM -c tmph1 ;; 1164esac 1165rm -rf dirTestDict 1166println "- dictionary builder on bogus input" 1167println "Hello World" > tmp 1168zstd --train-legacy -q tmp && die "Dictionary training should fail : not enough input source" 1169datagen -P0 -g10M > tmp 1170zstd --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise" 1171println "- Test -o before --train" 1172rm -f tmpDict dictionary 1173zstd -o tmpDict --train "$TESTDIR"/*.c "$PRGDIR"/*.c 1174test -f tmpDict 1175zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c 1176test -f dictionary 1177if [ -n "$hasMT" ] 1178then 1179 println "- Create dictionary with multithreading enabled" 1180 zstd --train -T0 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 1181fi 1182rm -f tmp* dictionary 1183 1184println "- Test --memory for dictionary compression" 1185datagen -g12M -P90 > tmpCorpusHighCompress 1186zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)" 1187zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr 1188cat zstTrainWithMemLimitStdErr | $GREP "setting manual memory limit for dictionary training data at 5 MB" 1189cat zstTrainWithMemLimitStdErr | $GREP "Training samples set too large (12 MB); training on 5 MB only..." 1190rm zstTrainWithMemLimitStdErr 1191 1192println "\n===> fastCover dictionary builder : advanced options " 1193TESTFILE="$PRGDIR"/zstdcli.c 1194datagen > tmpDict 1195println "- Create first dictionary" 1196zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 1197cp "$TESTFILE" tmp 1198zstd -f tmp -D tmpDict 1199zstd -d tmp.zst -D tmpDict -fo result 1200$DIFF "$TESTFILE" result 1201println "- Create second (different) dictionary" 1202zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 1203zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 1204zstd --train-fastcover=k=56,d=8 && die "Create dictionary without input file" 1205println "- Create dictionary with short dictID" 1206zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 1207cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 1208println "- Create dictionaries with shrink-dict flag enabled" 1209zstd --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict 1210zstd --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1 1211zstd --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2 1212zstd --train-fastcover=shrink=5,steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict3 1213println "- Create dictionary with size limit" 1214zstd --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K 1215println "- Create dictionary using all samples for both training and testing" 1216zstd --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1217println "- Create dictionary using f=16" 1218zstd --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1219zstd --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15" 1220println "- Create dictionary using accel=2" 1221zstd --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1222println "- Create dictionary using accel=10" 1223zstd --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1224println "- Create dictionary with multithreading" 1225zstd --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1226println "- Test -o before --train-fastcover" 1227rm -f tmpDict dictionary 1228zstd -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c 1229test -f tmpDict 1230zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c 1231test -f dictionary 1232rm -f tmp* dictionary 1233 1234 1235println "\n===> legacy dictionary builder " 1236 1237TESTFILE="$PRGDIR"/zstdcli.c 1238datagen > tmpDict 1239println "- Create first dictionary" 1240zstd --train-legacy=selectivity=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 1241cp "$TESTFILE" tmp 1242zstd -f tmp -D tmpDict 1243zstd -d tmp.zst -D tmpDict -fo result 1244$DIFF "$TESTFILE" result 1245zstd --train-legacy=s=8 && die "Create dictionary without input files (should error)" 1246println "- Create second (different) dictionary" 1247zstd --train-legacy=s=5 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 1248zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 1249println "- Create dictionary with short dictID" 1250zstd --train-legacy -s5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 1251cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 1252println "- Create dictionary with size limit" 1253zstd --train-legacy -s9 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K 1254println "- Test -o before --train-legacy" 1255rm -f tmpDict dictionary 1256zstd -o tmpDict --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c 1257test -f tmpDict 1258zstd --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c 1259test -f dictionary 1260rm -f tmp* dictionary 1261 1262 1263println "\n===> integrity tests " 1264 1265println "test one file (tmp1.zst) " 1266datagen > tmp1 1267zstd tmp1 1268zstd -t tmp1.zst 1269zstd --test tmp1.zst 1270println "test multiple files (*.zst) " 1271zstd -t ./*.zst 1272println "test bad files (*) " 1273zstd -t ./* && die "bad files not detected !" 1274zstd -t tmp1 && die "bad file not detected !" 1275cp tmp1 tmp2.zst 1276zstd -t tmp2.zst && die "bad file not detected !" 1277datagen -g0 > tmp3 1278zstd -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad 1279println "test --rm and --test combined " 1280zstd -t --rm tmp1.zst 1281test -f tmp1.zst # check file is still present 1282cp tmp1.zst tmp2.zst 1283zstd -t tmp1.zst tmp2.zst --rm 1284test -f tmp1.zst # check file is still present 1285test -f tmp2.zst # check file is still present 1286split -b16384 tmp1.zst tmpSplit. 1287zstd -t tmpSplit.* && die "bad file not detected !" 1288datagen | zstd -c | zstd -t 1289 1290 1291println "\n===> golden files tests " 1292 1293zstd -t -r "$TESTDIR/golden-decompression" 1294zstd -c -r "$TESTDIR/golden-compression" | zstd -t 1295zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" "$TESTDIR/golden-compression/http" -c | zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" -t 1296 1297 1298println "\n===> benchmark mode tests " 1299 1300println "bench one file" 1301datagen > tmp1 1302zstd -bi0 tmp1 1303println "bench multiple levels" 1304zstd -i0b0e3 tmp1 1305println "bench negative level" 1306zstd -bi0 --fast tmp1 1307println "with recursive and quiet modes" 1308zstd -rqi0b1e2 tmp1 1309println "benchmark decompression only" 1310zstd -f tmp1 1311zstd -b -d -i0 tmp1.zst 1312println "benchmark can fail - decompression on invalid data" 1313zstd -b -d -i0 tmp1 && die "invalid .zst data => benchmark should have failed" 1314 1315GZIPMODE=1 1316zstd --format=gzip -V || GZIPMODE=0 1317if [ $GZIPMODE -eq 1 ]; then 1318 println "benchmark mode is only compatible with zstd" 1319 zstd --format=gzip -b tmp1 && die "-b should be incompatible with gzip format!" 1320fi 1321 1322println "\n===> zstd compatibility tests " 1323 1324datagen > tmp 1325rm -f tmp.zst 1326zstd --format=zstd -f tmp 1327test -f tmp.zst 1328 1329 1330println "\n===> gzip compatibility tests " 1331 1332GZIPMODE=1 1333zstd --format=gzip -V || GZIPMODE=0 1334if [ $GZIPMODE -eq 1 ]; then 1335 println "gzip support detected" 1336 GZIPEXE=1 1337 gzip -V || GZIPEXE=0 1338 if [ $GZIPEXE -eq 1 ]; then 1339 datagen > tmp 1340 zstd --format=gzip -f tmp 1341 gzip -t -v tmp.gz 1342 gzip -f tmp 1343 zstd -d -f -v tmp.gz 1344 rm -f tmp* 1345 else 1346 println "gzip binary not detected" 1347 fi 1348else 1349 println "gzip mode not supported" 1350fi 1351 1352 1353println "\n===> gzip frame tests " 1354 1355if [ $GZIPMODE -eq 1 ]; then 1356 datagen > tmp 1357 zstd -f --format=gzip tmp 1358 zstd -f tmp 1359 cat tmp.gz tmp.zst tmp.gz tmp.zst | zstd -d -f -o tmp 1360 truncateLastByte tmp.gz | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1361 rm -f tmp* 1362else 1363 println "gzip mode not supported" 1364fi 1365 1366if [ $GZIPMODE -eq 1 ]; then 1367 datagen > tmp 1368 rm -f tmp.zst 1369 zstd --format=gzip --format=zstd -f tmp 1370 test -f tmp.zst 1371fi 1372 1373println "\n===> xz compatibility tests " 1374 1375LZMAMODE=1 1376zstd --format=xz -V || LZMAMODE=0 1377if [ $LZMAMODE -eq 1 ]; then 1378 println "xz support detected" 1379 XZEXE=1 1380 xz -Q -V && lzma -Q -V || XZEXE=0 1381 if [ $XZEXE -eq 1 ]; then 1382 println "Testing zstd xz and lzma support" 1383 datagen > tmp 1384 zstd --format=lzma -f tmp 1385 zstd --format=xz -f tmp 1386 xz -Q -t -v tmp.xz 1387 xz -Q -t -v tmp.lzma 1388 xz -Q -f -k tmp 1389 lzma -Q -f -k --lzma1 tmp 1390 zstd -d -f -v tmp.xz 1391 zstd -d -f -v tmp.lzma 1392 rm -f tmp* 1393 println "Creating symlinks" 1394 ln -s "$ZSTD_BIN" ./xz 1395 ln -s "$ZSTD_BIN" ./unxz 1396 ln -s "$ZSTD_BIN" ./lzma 1397 ln -s "$ZSTD_BIN" ./unlzma 1398 println "Testing xz and lzma symlinks" 1399 datagen > tmp 1400 ./xz tmp 1401 xz -Q -d tmp.xz 1402 ./lzma tmp 1403 lzma -Q -d tmp.lzma 1404 println "Testing unxz and unlzma symlinks" 1405 xz -Q tmp 1406 ./xz -d tmp.xz 1407 lzma -Q tmp 1408 ./lzma -d tmp.lzma 1409 rm -f xz unxz lzma unlzma 1410 rm -f tmp* 1411 else 1412 println "xz binary not detected" 1413 fi 1414else 1415 println "xz mode not supported" 1416fi 1417 1418 1419println "\n===> xz frame tests " 1420 1421if [ $LZMAMODE -eq 1 ]; then 1422 datagen > tmp 1423 zstd -f --format=xz tmp 1424 zstd -f --format=lzma tmp 1425 zstd -f tmp 1426 cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | zstd -d -f -o tmp 1427 truncateLastByte tmp.xz | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1428 truncateLastByte tmp.lzma | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1429 rm -f tmp* 1430else 1431 println "xz mode not supported" 1432fi 1433 1434println "\n===> lz4 compatibility tests " 1435 1436LZ4MODE=1 1437zstd --format=lz4 -V || LZ4MODE=0 1438if [ $LZ4MODE -eq 1 ]; then 1439 println "lz4 support detected" 1440 LZ4EXE=1 1441 lz4 -V || LZ4EXE=0 1442 if [ $LZ4EXE -eq 1 ]; then 1443 datagen > tmp 1444 zstd --format=lz4 -f tmp 1445 lz4 -t -v tmp.lz4 1446 lz4 -f -m tmp # ensure result is sent into tmp.lz4, not stdout 1447 zstd -d -f -v tmp.lz4 1448 rm -f tmp* 1449 else 1450 println "lz4 binary not detected" 1451 fi 1452else 1453 println "lz4 mode not supported" 1454fi 1455 1456 1457if [ $LZ4MODE -eq 1 ]; then 1458 println "\n===> lz4 frame tests " 1459 datagen > tmp 1460 zstd -f --format=lz4 tmp 1461 zstd -f tmp 1462 cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | zstd -d -f -o tmp 1463 truncateLastByte tmp.lz4 | zstd -t > $INTOVOID && die "incomplete frame not detected !" 1464 rm -f tmp* 1465else 1466 println "\nlz4 mode not supported" 1467fi 1468 1469 1470println "\n===> suffix list test" 1471 1472! zstd -d tmp.abc 2> tmplg 1473 1474if [ $GZIPMODE -ne 1 ]; then 1475 $GREP ".gz" tmplg > $INTOVOID && die "Unsupported suffix listed" 1476fi 1477 1478if [ $LZMAMODE -ne 1 ]; then 1479 $GREP ".lzma" tmplg > $INTOVOID && die "Unsupported suffix listed" 1480 $GREP ".xz" tmplg > $INTOVOID && die "Unsupported suffix listed" 1481fi 1482 1483if [ $LZ4MODE -ne 1 ]; then 1484 $GREP ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed" 1485fi 1486 1487touch tmp1 1488zstd tmp1 -o tmp1.zstd 1489zstd -d -f tmp1.zstd # support .zstd suffix even though it's not the default suffix 1490 1491println "\n===> tar extension tests " 1492 1493rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 tmp1.zstd 1494 1495datagen > tmp 1496tar -cf tmp.tar tmp 1497zstd tmp.tar -o tmp.tzst 1498rm -f tmp.tar 1499zstd -d tmp.tzst 1500[ -e tmp.tar ] || die ".tzst failed to decompress to .tar!" 1501rm -f tmp.tar tmp.tzst 1502 1503if [ $GZIPMODE -eq 1 ]; then 1504 tar -f - -c tmp | gzip > tmp.tgz 1505 zstd -d tmp.tgz 1506 [ -e tmp.tar ] || die ".tgz failed to decompress to .tar!" 1507 rm -f tmp.tar tmp.tgz 1508fi 1509 1510if [ $LZMAMODE -eq 1 ]; then 1511 tar -f - -c tmp | zstd --format=xz > tmp.txz 1512 zstd -d tmp.txz 1513 [ -e tmp.tar ] || die ".txz failed to decompress to .tar!" 1514 rm -f tmp.tar tmp.txz 1515fi 1516 1517if [ $LZ4MODE -eq 1 ]; then 1518 tar -f - -c tmp | zstd --format=lz4 > tmp.tlz4 1519 zstd -d tmp.tlz4 1520 [ -e tmp.tar ] || die ".tlz4 failed to decompress to .tar!" 1521 rm -f tmp.tar tmp.tlz4 1522fi 1523 1524touch tmp.t tmp.tz tmp.tzs 1525! zstd -d tmp.t 1526! zstd -d tmp.tz 1527! zstd -d tmp.tzs 1528 1529 1530println "\n===> zstd round-trip tests " 1531 1532roundTripTest 1533roundTripTest -g15K # TableID==3 1534roundTripTest -g127K # TableID==2 1535roundTripTest -g255K # TableID==1 1536roundTripTest -g522K # TableID==0 1537roundTripTest -g519K 6 # greedy, hash chain 1538roundTripTest -g517K 16 # btlazy2 1539roundTripTest -g516K 19 # btopt 1540 1541fileRoundTripTest -g500K 1542 1543println "\n===> zstd long distance matching round-trip tests " 1544roundTripTest -g0 "2 --single-thread --long" 1545roundTripTest -g1000K "1 --single-thread --long" 1546roundTripTest -g517K "6 --single-thread --long" 1547roundTripTest -g516K "16 --single-thread --long" 1548roundTripTest -g518K "19 --single-thread --long" 1549roundTripTest -g2M "22 --single-thread --ultra --long" 1550fileRoundTripTest -g5M "3 --single-thread --long" 1551 1552 1553roundTripTest -g96K "5 --single-thread" 1554if [ -n "$hasMT" ] 1555then 1556 println "\n===> zstdmt round-trip tests " 1557 roundTripTest -g4M "1 -T0" 1558 roundTripTest -g4M "1 -T0 --auto-threads=physical" 1559 roundTripTest -g4M "1 -T0 --auto-threads=logical" 1560 roundTripTest -g8M "3 -T2" 1561 roundTripTest -g8M "19 --long" 1562 roundTripTest -g8000K "2 --threads=2" 1563 fileRoundTripTest -g4M "19 -T2 -B1M" 1564 1565 println "\n===> zstdmt long distance matching round-trip tests " 1566 roundTripTest -g8M "3 --long=24 -T2" 1567 1568 println "\n===> zstdmt environment variable tests " 1569 echo "multifoo" >> mt_tmp 1570 ZSTD_NBTHREADS=-3 zstd -f mt_tmp # negative value, warn and revert to default setting 1571 ZSTD_NBTHREADS='' zstd -f mt_tmp # empty env var, warn and revert to default setting 1572 ZSTD_NBTHREADS=- zstd -f mt_tmp # malformed env var, warn and revert to default setting 1573 ZSTD_NBTHREADS=a zstd -f mt_tmp # malformed env var, warn and revert to default setting 1574 ZSTD_NBTHREADS=+a zstd -f mt_tmp # malformed env var, warn and revert to default setting 1575 ZSTD_NBTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting 1576 ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting= 1577 ZSTD_NBTHREADS=2 zstd -f mt_tmp # correct usage 1578 ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single thread 1579 # temporary envvar changes in the above tests would actually persist in macos /bin/sh 1580 unset ZSTD_NBTHREADS 1581 rm -f mt_tmp* 1582 1583 println "\n===> ovLog tests " 1584 datagen -g2MB > tmp 1585 refSize=$(zstd tmp -6 -c --zstd=wlog=18 | wc -c) 1586 ov9Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c) 1587 ov1Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c) 1588 if [ "$refSize" -eq "$ov9Size" ]; then 1589 echo ov9Size should be different from refSize 1590 exit 1 1591 fi 1592 if [ "$refSize" -eq "$ov1Size" ]; then 1593 echo ov1Size should be different from refSize 1594 exit 1 1595 fi 1596 if [ "$ov9Size" -ge "$ov1Size" ]; then 1597 echo ov9Size="$ov9Size" should be smaller than ov1Size="$ov1Size" 1598 exit 1 1599 fi 1600 1601else 1602 println "\n===> no multithreading, skipping zstdmt tests " 1603fi 1604 1605rm -f tmp* 1606 1607println "\n===> zstd --list/-l single frame tests " 1608datagen > tmp1 1609datagen > tmp2 1610datagen > tmp3 1611zstd tmp* 1612zstd -l ./*.zst 1613zstd -lv ./*.zst | $GREP "Decompressed Size:" # check that decompressed size is present in header 1614zstd --list ./*.zst 1615zstd --list -v ./*.zst 1616 1617println "\n===> zstd --list/-l multiple frame tests " 1618cat tmp1.zst tmp2.zst > tmp12.zst 1619cat tmp12.zst tmp3.zst > tmp123.zst 1620zstd -l ./*.zst 1621zstd -lv ./*.zst 1622 1623println "\n===> zstd --list/-l error detection tests " 1624zstd -l tmp1 tmp1.zst && die "-l must fail on non-zstd file" 1625zstd --list tmp* && die "-l must fail on non-zstd file" 1626zstd -lv tmp1* && die "-l must fail on non-zstd file" 1627zstd --list -v tmp2 tmp12.zst && die "-l must fail on non-zstd file" 1628 1629println "test : detect truncated compressed file " 1630TEST_DATA_FILE=truncatable-input.txt 1631FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst 1632TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst 1633datagen -g50000 > $TEST_DATA_FILE 1634zstd -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE 1635dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE 1636zstd --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file" 1637 1638rm -f $TEST_DATA_FILE 1639rm -f $FULL_COMPRESSED_FILE 1640rm -f $TRUNCATED_COMPRESSED_FILE 1641 1642println "\n===> zstd --list/-l errors when presented with stdin / no files" 1643zstd -l && die "-l must fail on empty list of files" 1644zstd -l - && die "-l does not work on stdin" 1645zstd -l < tmp1.zst && die "-l does not work on stdin" 1646zstd -l - < tmp1.zst && die "-l does not work on stdin" 1647zstd -l - tmp1.zst && die "-l does not work on stdin" 1648zstd -l - tmp1.zst < tmp1.zst && die "-l does not work on stdin" 1649zstd -l tmp1.zst < tmp2.zst # this will check tmp1.zst, but not tmp2.zst, which is not an error : zstd simply doesn't read stdin in this case. It must not error just because stdin is not a tty 1650 1651println "\n===> zstd --list/-l test with null files " 1652datagen -g0 > tmp5 1653zstd tmp5 1654zstd -l tmp5.zst 1655zstd -l tmp5* && die "-l must fail on non-zstd file" 1656zstd -lv tmp5.zst | $GREP "Decompressed Size: 0 B (0 B)" # check that 0 size is present in header 1657zstd -lv tmp5* && die "-l must fail on non-zstd file" 1658 1659println "\n===> zstd --list/-l test with no content size field " 1660datagen -g513K | zstd > tmp6.zst 1661zstd -l tmp6.zst 1662zstd -lv tmp6.zst | $GREP "Decompressed Size:" && die "Field :Decompressed Size: should not be available in this compressed file" 1663 1664println "\n===> zstd --list/-l test with no checksum " 1665zstd -f --no-check tmp1 1666zstd -l tmp1.zst 1667zstd -lv tmp1.zst 1668 1669println "\n===> zstd trace tests " 1670zstd -f --trace tmp.trace tmp1 1671zstd -f --trace tmp.trace tmp1 tmp2 tmp3 1672zstd -f --trace tmp.trace tmp1 tmp2 tmp3 -o /dev/null 1673zstd -f --trace tmp.trace tmp1 tmp2 tmp3 --single-thread 1674zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null 1675zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null --single-thread 1676zstd --trace tmp.trace -t tmp1.zst 1677zstd --trace tmp.trace -t tmp1.zst tmp2.zst 1678zstd -f --trace tmp.trace -d tmp1.zst 1679zstd -f --trace tmp.trace -d tmp1.zst tmp2.zst tmp3.zst 1680zstd -D tmp1 tmp2 -c | zstd --trace tmp.trace -t -D tmp1 1681zstd -b1e10i0 --trace tmp.trace tmp1 1682zstd -b1e10i0 --trace tmp.trace tmp1 tmp2 tmp3 1683 1684rm -f tmp* 1685 1686 1687println "\n===> zstd long distance matching tests " 1688roundTripTest -g0 " --single-thread --long" 1689roundTripTest -g9M "2 --single-thread --long" 1690# Test parameter parsing 1691roundTripTest -g1M -P50 "1 --single-thread --long=29" " --memory=512MB" 1692roundTripTest -g1M -P50 "1 --single-thread --long=29 --zstd=wlog=28" " --memory=256MB" 1693roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB" 1694roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB" 1695 1696 1697if [ "$ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP" -ne "1" ]; then 1698 println "\n===> zstd long distance matching with optimal parser compressed size tests " 1699 optCSize16=$(datagen -g511K | zstd -16 -c | wc -c) 1700 longCSize16=$(datagen -g511K | zstd -16 --long -c | wc -c) 1701 optCSize19=$(datagen -g2M | zstd -19 -c | wc -c) 1702 longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c) 1703 optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c) 1704 longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c) 1705 if [ "$longCSize16" -gt "$optCSize16" ]; then 1706 echo using --long on compression level 16 should not cause compressed size regression 1707 exit 1 1708 elif [ "$longCSize19" -gt "$optCSize19" ]; then 1709 echo using --long on compression level 19 should not cause compressed size regression 1710 exit 1 1711 elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then 1712 echo using --long on compression level 19 with wLog=23 should not cause compressed size regression 1713 exit 1 1714 fi 1715fi 1716 1717println "\n===> zstd asyncio tests " 1718 1719addFrame() { 1720 datagen -g2M -s$2 >> tmp_uncompressed 1721 datagen -g2M -s$2 | zstd -1 --format=$1 >> tmp_compressed.zst 1722} 1723 1724addTwoFrames() { 1725 addFrame $1 1 1726 addFrame $1 2 1727} 1728 1729testAsyncIO() { 1730 roundTripTest -g2M "3 --asyncio --format=$1" 1731 roundTripTest -g2M "3 --no-asyncio --format=$1" 1732} 1733 1734rm -f tmp_compressed tmp_uncompressed 1735testAsyncIO zstd 1736addTwoFrames zstd 1737if [ $GZIPMODE -eq 1 ]; then 1738 testAsyncIO gzip 1739 addTwoFrames gzip 1740fi 1741if [ $LZMAMODE -eq 1 ]; then 1742 testAsyncIO lzma 1743 addTwoFrames lzma 1744fi 1745if [ $LZ4MODE -eq 1 ]; then 1746 testAsyncIO lz4 1747 addTwoFrames lz4 1748fi 1749cat tmp_uncompressed | $MD5SUM > tmp2 1750zstd -d tmp_compressed.zst --asyncio -c | $MD5SUM > tmp1 1751$DIFF -q tmp1 tmp2 1752rm tmp1 1753zstd -d tmp_compressed.zst --no-asyncio -c | $MD5SUM > tmp1 1754$DIFF -q tmp1 tmp2 1755 1756if [ "$1" != "--test-large-data" ]; then 1757 println "Skipping large data tests" 1758 exit 0 1759fi 1760 1761 1762############################################################################# 1763 1764 1765if [ -n "$hasMT" ] 1766then 1767 println "\n===> adaptive mode " 1768 roundTripTest -g270000000 " --adapt" 1769 roundTripTest -g27000000 " --adapt=min=1,max=4" 1770 roundTripTest -g27000000 " --adapt=min=-2,max=-1" 1771 println "===> test: --adapt must fail on incoherent bounds " 1772 datagen > tmp 1773 zstd --adapt= tmp && die "invalid compression parameter" 1774 zstd -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds" 1775 1776 println "\n===> rsyncable mode " 1777 roundTripTest -g10M " --rsyncable" 1778 roundTripTest -g10M " --rsyncable -B100K" 1779 println "===> test: --rsyncable must fail with --single-thread" 1780 zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread" 1781fi 1782 1783println "\n===> patch-from=origin tests" 1784datagen -g1000 -P50 > tmp_dict 1785datagen -g1000 -P10 > tmp_patch 1786zstd --patch-from=tmp_dict tmp_patch -o tmp_patch_diff 1787zstd -d --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon 1788$DIFF -s tmp_patch_recon tmp_patch 1789 1790println "\n===> alternate syntax: patch-from origin" 1791zstd -f --patch-from tmp_dict tmp_patch -o tmp_patch_diff 1792zstd -df --patch-from tmp_dict tmp_patch_diff -o tmp_patch_recon 1793$DIFF -s tmp_patch_recon tmp_patch 1794rm -rf tmp_* 1795 1796println "\n===> patch-from recursive tests" 1797mkdir tmp_dir 1798datagen > tmp_dir/tmp1 1799datagen > tmp_dir/tmp2 1800datagen > tmp_dict 1801zstd --patch-from=tmp_dict -r tmp_dir && die 1802rm -rf tmp* 1803 1804println "\n===> patch-from long mode trigger larger file test" 1805if [ "$ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP" -eq "1" ]; then 1806 # if binary tree strategies are excluded, the threshold is different 1807 datagen -g10000000 > tmp_dict 1808 datagen -g10000000 > tmp_patch 1809else 1810 datagen -g5000000 > tmp_dict 1811 datagen -g5000000 > tmp_patch 1812fi 1813zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | $GREP "long mode automatically triggered" 1814rm -rf tmp* 1815 1816println "\n===> patch-from very large dictionary and file test" 1817datagen -g550000000 -P0 > tmp_dict 1818datagen -g100000000 -P1 > tmp_patch 1819zstd --long=30 -1f --patch-from tmp_dict tmp_patch 1820zstd --long=30 -df --patch-from tmp_dict tmp_patch.zst -o tmp_patch_recon 1821$DIFF -s tmp_patch_recon tmp_patch 1822rm -rf tmp* 1823 1824println "\n===> patch-from --stream-size test" 1825datagen -g1000 -P50 > tmp_dict 1826datagen -g1000 -P10 > tmp_patch 1827cat tmp_patch | zstd -f --patch-from=tmp_dict -c -o tmp_patch_diff && die 1828cat tmp_patch | zstd -f --patch-from=tmp_dict --stream-size=1000 -c -o tmp_patch_diff 1829rm -rf tmp* 1830 1831println "\n===> large files tests " 1832 1833roundTripTest -g270000000 1 1834roundTripTest -g250000000 2 1835roundTripTest -g230000000 3 1836 1837roundTripTest -g140000000 -P60 4 1838roundTripTest -g130000000 -P62 5 1839roundTripTest -g120000000 -P65 6 1840 1841roundTripTest -g70000000 -P70 7 1842roundTripTest -g60000000 -P71 8 1843roundTripTest -g50000000 -P73 9 1844 1845roundTripTest -g35000000 -P75 10 1846roundTripTest -g30000000 -P76 11 1847roundTripTest -g25000000 -P78 12 1848 1849roundTripTest -g18000013 -P80 13 1850roundTripTest -g18000014 -P80 14 1851roundTripTest -g18000015 -P81 15 1852roundTripTest -g18000016 -P84 16 1853roundTripTest -g18000017 -P88 17 1854roundTripTest -g18000018 -P94 18 1855roundTripTest -g18000019 -P96 19 1856 1857roundTripTest -g5000000000 -P99 "1 --zstd=wlog=25" 1858roundTripTest -g3700000000 -P0 "1 --zstd=strategy=6,wlog=25" # ensure btlazy2 can survive an overflow rescale 1859 1860fileRoundTripTest -g4193M -P99 1 1861 1862 1863println "\n===> zstd long, long distance matching round-trip tests " 1864roundTripTest -g270000000 "1 --single-thread --long" 1865roundTripTest -g130000000 -P60 "5 --single-thread --long" 1866roundTripTest -g35000000 -P70 "8 --single-thread --long" 1867roundTripTest -g18000001 -P80 "18 --single-thread --long" 1868# Test large window logs 1869roundTripTest -g700M -P50 "1 --single-thread --long=29" 1870roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28" 1871 1872 1873if [ -n "$hasMT" ] 1874then 1875 println "\n===> zstdmt long round-trip tests " 1876 roundTripTest -g80000000 -P99 "19 -T2" " " 1877 roundTripTest -g5000000000 -P99 "1 -T2" " " 1878 roundTripTest -g500000000 -P97 "1 -T999" " " 1879 fileRoundTripTest -g4103M -P98 " -T0" " " 1880 roundTripTest -g400000000 -P97 "1 --long=24 -T2" " " 1881 # Exposes the bug in https://github.com/facebook/zstd/pull/1678 1882 # This test fails on 4 different travis builds at the time of writing 1883 # because it needs to allocate 8 GB of memory. 1884 # roundTripTest -g10G -P99 "1 -T1 --long=31 --zstd=clog=27 --fast=1000" 1885else 1886 println "\n**** no multithreading, skipping zstdmt tests **** " 1887fi 1888 1889 1890println "\n===> cover dictionary builder : advanced options " 1891 1892TESTFILE="$PRGDIR"/zstdcli.c 1893datagen > tmpDict 1894println "- Create first dictionary" 1895zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict 1896cp "$TESTFILE" tmp 1897zstd -f tmp -D tmpDict 1898zstd -f tmp -D tmpDict --patch-from=tmpDict && die "error: can't use -D and --patch-from=#at the same time" 1899zstd -d tmp.zst -D tmpDict -fo result 1900$DIFF "$TESTFILE" result 1901zstd --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)" 1902println "- Create second (different) dictionary" 1903zstd --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC 1904zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" 1905println "- Create dictionary using shrink-dict flag" 1906zstd --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict 1907zstd --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1 1908zstd --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2 1909zstd --train-cover=shrink=5,steps=256 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict3 1910println "- Create dictionary with short dictID" 1911zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 1912cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" 1913println "- Create dictionary with size limit" 1914zstd --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K 1915println "- Compare size of dictionary from 90% training samples with 80% training samples" 1916zstd --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1917zstd --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1918println "- Create dictionary using all samples for both training and testing" 1919zstd --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c 1920println "- Test -o before --train-cover" 1921rm -f tmpDict dictionary 1922zstd -o tmpDict --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c 1923test -f tmpDict 1924zstd --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c 1925test -f dictionary 1926rm -f tmp* dictionary 1927 1928rm -f tmp* 1929