dropshell release 2025.0513.2134
Some checks failed
Dropshell Test / Build_and_Test (push) Has been cancelled
Some checks failed
Dropshell Test / Build_and_Test (push) Has been cancelled
This commit is contained in:
Your Name
72
build_amd64/_deps/zstd-src/tests/.gitignore
vendored
Normal file
72
build_amd64/_deps/zstd-src/tests/.gitignore
vendored
Normal file
@@ -0,0 +1,72 @@
|
||||
# local binary (Makefile)
|
||||
fullbench
|
||||
fullbench32
|
||||
fullbench-lib
|
||||
fuzzer
|
||||
fuzzer32
|
||||
fuzzer-dll
|
||||
zbufftest
|
||||
zbufftest32
|
||||
zbufftest-dll
|
||||
zstreamtest
|
||||
zstreamtest32
|
||||
zstreamtest_asan
|
||||
zstreamtest_tsan
|
||||
zstreamtest_ubsan
|
||||
zstreamtest-dll
|
||||
datagen
|
||||
paramgrill
|
||||
paramgrill32
|
||||
roundTripCrash
|
||||
longmatch
|
||||
symbols
|
||||
legacy
|
||||
decodecorpus
|
||||
pool
|
||||
poolTests
|
||||
invalidDictionaries
|
||||
checkTag
|
||||
zcat
|
||||
zstdcat
|
||||
tm
|
||||
|
||||
# test artifacts
|
||||
dictionary
|
||||
grillResults.txt
|
||||
_*
|
||||
tmp*
|
||||
*.zst
|
||||
*.gz
|
||||
!gzip/hufts-segv.gz
|
||||
result
|
||||
out
|
||||
*.zstd
|
||||
hello*
|
||||
world
|
||||
|
||||
# Tmp test directory
|
||||
zstdtest
|
||||
speedTest
|
||||
versionsTest
|
||||
namespaceTest
|
||||
dirTest*
|
||||
|
||||
# fuzzer
|
||||
afl
|
||||
|
||||
# Local script
|
||||
startSpeedTest
|
||||
speedTest.pid
|
||||
*.bat
|
||||
|
||||
# Generic Object files
|
||||
*.o
|
||||
*.ko
|
||||
|
||||
# Generic Executables
|
||||
*.exe
|
||||
*.out
|
||||
*.app
|
||||
|
||||
# Specific exclusions
|
||||
!golden-decompression/*.zst
|
||||
378
build_amd64/_deps/zstd-src/tests/DEPRECATED-test-zstd-speed.py
Executable file
378
build_amd64/_deps/zstd-src/tests/DEPRECATED-test-zstd-speed.py
Executable file
@@ -0,0 +1,378 @@
|
||||
#! /usr/bin/env python3
|
||||
# THIS BENCHMARK IS BEING REPLACED BY automated-bencmarking.py
|
||||
|
||||
# ################################################################
|
||||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# You may select, at your option, one of the above-listed licenses.
|
||||
# ##########################################################################
|
||||
|
||||
# Limitations:
|
||||
# - doesn't support filenames with spaces
|
||||
# - dir1/zstd and dir2/zstd will be merged in a single results file
|
||||
|
||||
import argparse
|
||||
import os # getloadavg
|
||||
import string
|
||||
import subprocess
|
||||
import time # strftime
|
||||
import traceback
|
||||
import hashlib
|
||||
import platform # system
|
||||
|
||||
script_version = 'v1.1.2 (2017-03-26)'
|
||||
default_repo_url = 'https://github.com/facebook/zstd.git'
|
||||
working_dir_name = 'speedTest'
|
||||
working_path = os.getcwd() + '/' + working_dir_name # /path/to/zstd/tests/speedTest
|
||||
clone_path = working_path + '/' + 'zstd' # /path/to/zstd/tests/speedTest/zstd
|
||||
email_header = 'ZSTD_speedTest'
|
||||
pid = str(os.getpid())
|
||||
verbose = False
|
||||
clang_version = "unknown"
|
||||
gcc_version = "unknown"
|
||||
args = None
|
||||
|
||||
|
||||
def hashfile(hasher, fname, blocksize=65536):
|
||||
with open(fname, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(blocksize), b""):
|
||||
hasher.update(chunk)
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def log(text):
|
||||
print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text)
|
||||
|
||||
|
||||
def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True):
|
||||
if print_command:
|
||||
log("> " + command)
|
||||
popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell, cwd=execute.cwd)
|
||||
stdout_lines, stderr_lines = popen.communicate(timeout=args.timeout)
|
||||
stderr_lines = stderr_lines.decode("utf-8")
|
||||
stdout_lines = stdout_lines.decode("utf-8")
|
||||
if print_output:
|
||||
if stdout_lines:
|
||||
print(stdout_lines)
|
||||
if stderr_lines:
|
||||
print(stderr_lines)
|
||||
if popen.returncode is not None and popen.returncode != 0:
|
||||
if stderr_lines and not print_output and print_error:
|
||||
print(stderr_lines)
|
||||
raise RuntimeError(stdout_lines + stderr_lines)
|
||||
return (stdout_lines + stderr_lines).splitlines()
|
||||
execute.cwd = None
|
||||
|
||||
|
||||
def does_command_exist(command):
|
||||
try:
|
||||
execute(command, verbose, False, False)
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def send_email(emails, topic, text, have_mutt, have_mail):
|
||||
logFileName = working_path + '/' + 'tmpEmailContent'
|
||||
with open(logFileName, "w") as myfile:
|
||||
myfile.writelines(text)
|
||||
myfile.close()
|
||||
if have_mutt:
|
||||
execute('mutt -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose)
|
||||
elif have_mail:
|
||||
execute('mail -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose)
|
||||
else:
|
||||
log("e-mail cannot be sent (mail or mutt not found)")
|
||||
|
||||
|
||||
def send_email_with_attachments(branch, commit, last_commit, args, text, results_files,
|
||||
logFileName, have_mutt, have_mail):
|
||||
with open(logFileName, "w") as myfile:
|
||||
myfile.writelines(text)
|
||||
myfile.close()
|
||||
email_topic = '[%s:%s] Warning for %s:%s last_commit=%s speed<%s ratio<%s' \
|
||||
% (email_header, pid, branch, commit, last_commit,
|
||||
args.lowerLimit, args.ratioLimit)
|
||||
if have_mutt:
|
||||
execute('mutt -s "' + email_topic + '" ' + args.emails + ' -a ' + results_files
|
||||
+ ' < ' + logFileName)
|
||||
elif have_mail:
|
||||
execute('mail -s "' + email_topic + '" ' + args.emails + ' < ' + logFileName)
|
||||
else:
|
||||
log("e-mail cannot be sent (mail or mutt not found)")
|
||||
|
||||
|
||||
def git_get_branches():
|
||||
execute('git fetch -p', verbose)
|
||||
branches = execute('git branch -rl', verbose)
|
||||
output = []
|
||||
for line in branches:
|
||||
if ("HEAD" not in line) and ("coverity_scan" not in line) and ("gh-pages" not in line):
|
||||
output.append(line.strip())
|
||||
return output
|
||||
|
||||
|
||||
def git_get_changes(branch, commit, last_commit):
|
||||
fmt = '--format="%h: (%an) %s, %ar"'
|
||||
if last_commit is None:
|
||||
commits = execute('git log -n 10 %s %s' % (fmt, commit))
|
||||
else:
|
||||
commits = execute('git --no-pager log %s %s..%s' % (fmt, last_commit, commit))
|
||||
return str('Changes in %s since %s:\n' % (branch, last_commit)) + '\n'.join(commits)
|
||||
|
||||
|
||||
def get_last_results(resultsFileName):
|
||||
if not os.path.isfile(resultsFileName):
|
||||
return None, None, None, None
|
||||
commit = None
|
||||
csize = []
|
||||
cspeed = []
|
||||
dspeed = []
|
||||
with open(resultsFileName, 'r') as f:
|
||||
for line in f:
|
||||
words = line.split()
|
||||
if len(words) <= 4: # branch + commit + compilerVer + md5
|
||||
commit = words[1]
|
||||
csize = []
|
||||
cspeed = []
|
||||
dspeed = []
|
||||
if (len(words) == 8) or (len(words) == 9): # results: "filename" or "XX files"
|
||||
csize.append(int(words[1]))
|
||||
cspeed.append(float(words[3]))
|
||||
dspeed.append(float(words[5]))
|
||||
return commit, csize, cspeed, dspeed
|
||||
|
||||
|
||||
def benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName,
|
||||
testFilePath, fileName, last_csize, last_cspeed, last_dspeed):
|
||||
sleepTime = 30
|
||||
while os.getloadavg()[0] > args.maxLoadAvg:
|
||||
log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds"
|
||||
% (os.getloadavg()[0], args.maxLoadAvg, sleepTime))
|
||||
time.sleep(sleepTime)
|
||||
start_load = str(os.getloadavg())
|
||||
osType = platform.system()
|
||||
if osType == 'Linux':
|
||||
cpuSelector = "taskset --cpu-list 0"
|
||||
else:
|
||||
cpuSelector = ""
|
||||
if args.dictionary:
|
||||
result = execute('%s programs/%s -rqi5b1e%s -D %s %s' % (cpuSelector, executableName, args.lastCLevel, args.dictionary, testFilePath), print_output=True)
|
||||
else:
|
||||
result = execute('%s programs/%s -rqi5b1e%s %s' % (cpuSelector, executableName, args.lastCLevel, testFilePath), print_output=True)
|
||||
end_load = str(os.getloadavg())
|
||||
linesExpected = args.lastCLevel + 1
|
||||
if len(result) != linesExpected:
|
||||
raise RuntimeError("ERROR: number of result lines=%d is different that expected %d\n%s" % (len(result), linesExpected, '\n'.join(result)))
|
||||
with open(resultsFileName, "a") as myfile:
|
||||
myfile.write('%s %s %s md5=%s\n' % (branch, commit, compilerVersion, md5sum))
|
||||
myfile.write('\n'.join(result) + '\n')
|
||||
myfile.close()
|
||||
if (last_cspeed == None):
|
||||
log("WARNING: No data for comparison for branch=%s file=%s " % (branch, fileName))
|
||||
return ""
|
||||
commit, csize, cspeed, dspeed = get_last_results(resultsFileName)
|
||||
text = ""
|
||||
for i in range(0, min(len(cspeed), len(last_cspeed))):
|
||||
print("%s:%s -%d cSpeed=%6.2f cLast=%6.2f cDiff=%1.4f dSpeed=%6.2f dLast=%6.2f dDiff=%1.4f ratioDiff=%1.4f %s" % (branch, commit, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], float(last_csize[i])/csize[i], fileName))
|
||||
if (cspeed[i]/last_cspeed[i] < args.lowerLimit):
|
||||
text += "WARNING: %s -%d cSpeed=%.2f cLast=%.2f cDiff=%.4f %s\n" % (executableName, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], fileName)
|
||||
if (dspeed[i]/last_dspeed[i] < args.lowerLimit):
|
||||
text += "WARNING: %s -%d dSpeed=%.2f dLast=%.2f dDiff=%.4f %s\n" % (executableName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], fileName)
|
||||
if (float(last_csize[i])/csize[i] < args.ratioLimit):
|
||||
text += "WARNING: %s -%d cSize=%d last_cSize=%d diff=%.4f %s\n" % (executableName, i+1, csize[i], last_csize[i], float(last_csize[i])/csize[i], fileName)
|
||||
if text:
|
||||
text = args.message + ("\nmaxLoadAvg=%s load average at start=%s end=%s\n%s last_commit=%s md5=%s\n" % (args.maxLoadAvg, start_load, end_load, compilerVersion, last_commit, md5sum)) + text
|
||||
return text
|
||||
|
||||
|
||||
def update_config_file(branch, commit):
|
||||
last_commit = None
|
||||
commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt"
|
||||
if os.path.isfile(commitFileName):
|
||||
with open(commitFileName, 'r') as infile:
|
||||
last_commit = infile.read()
|
||||
with open(commitFileName, 'w') as outfile:
|
||||
outfile.write(commit)
|
||||
return last_commit
|
||||
|
||||
|
||||
def double_check(branch, commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName):
|
||||
last_commit, csize, cspeed, dspeed = get_last_results(resultsFileName)
|
||||
if not args.dry_run:
|
||||
text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
|
||||
if text:
|
||||
log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit))
|
||||
text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed)
|
||||
return text
|
||||
|
||||
|
||||
def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail):
|
||||
local_branch = branch.split('/')[1]
|
||||
version = local_branch.rpartition('-')[2] + '_' + commit
|
||||
if not args.dry_run:
|
||||
execute('make -C programs clean zstd CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -DZSTD_GIT_COMMIT=%s" && ' % version +
|
||||
'mv programs/zstd programs/zstd_clang && ' +
|
||||
'make -C programs clean zstd zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version)
|
||||
md5_zstd = hashfile(hashlib.md5(), clone_path + '/programs/zstd')
|
||||
md5_zstd32 = hashfile(hashlib.md5(), clone_path + '/programs/zstd32')
|
||||
md5_zstd_clang = hashfile(hashlib.md5(), clone_path + '/programs/zstd_clang')
|
||||
print("md5(zstd)=%s\nmd5(zstd32)=%s\nmd5(zstd_clang)=%s" % (md5_zstd, md5_zstd32, md5_zstd_clang))
|
||||
print("gcc_version=%s clang_version=%s" % (gcc_version, clang_version))
|
||||
|
||||
logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt"
|
||||
text_to_send = []
|
||||
results_files = ""
|
||||
if args.dictionary:
|
||||
dictName = args.dictionary.rpartition('/')[2]
|
||||
else:
|
||||
dictName = None
|
||||
|
||||
for filePath in testFilePaths:
|
||||
fileName = filePath.rpartition('/')[2]
|
||||
if dictName:
|
||||
resultsFileName = working_path + "/" + dictName.replace(".", "_") + "_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
|
||||
else:
|
||||
resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
|
||||
text = double_check(branch, commit, args, 'zstd', md5_zstd, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName)
|
||||
if text:
|
||||
text_to_send.append(text)
|
||||
results_files += resultsFileName + " "
|
||||
resultsFileName = working_path + "/results32_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
|
||||
text = double_check(branch, commit, args, 'zstd32', md5_zstd32, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName)
|
||||
if text:
|
||||
text_to_send.append(text)
|
||||
results_files += resultsFileName + " "
|
||||
resultsFileName = working_path + "/resultsClang_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt"
|
||||
text = double_check(branch, commit, args, 'zstd_clang', md5_zstd_clang, 'clang_version='+clang_version, resultsFileName, filePath, fileName)
|
||||
if text:
|
||||
text_to_send.append(text)
|
||||
results_files += resultsFileName + " "
|
||||
if text_to_send:
|
||||
send_email_with_attachments(branch, commit, last_commit, args, text_to_send, results_files, logFileName, have_mutt, have_mail)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('testFileNames', help='file or directory names list for speed benchmark')
|
||||
parser.add_argument('emails', help='list of e-mail addresses to send warnings')
|
||||
parser.add_argument('--dictionary', '-D', help='path to the dictionary')
|
||||
parser.add_argument('--message', '-m', help='attach an additional message to e-mail', default="")
|
||||
parser.add_argument('--repoURL', help='changes default repository URL', default=default_repo_url)
|
||||
parser.add_argument('--lowerLimit', '-l', type=float, help='send email if speed is lower than given limit', default=0.98)
|
||||
parser.add_argument('--ratioLimit', '-r', type=float, help='send email if ratio is lower than given limit', default=0.999)
|
||||
parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75)
|
||||
parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5)
|
||||
parser.add_argument('--sleepTime', '-s', type=int, help='frequency of repository checking in seconds', default=300)
|
||||
parser.add_argument('--timeout', '-t', type=int, help='timeout for executing shell commands', default=1800)
|
||||
parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False)
|
||||
parser.add_argument('--verbose', '-v', action='store_true', help='more verbose logs', default=False)
|
||||
args = parser.parse_args()
|
||||
verbose = args.verbose
|
||||
|
||||
# check if test files are accessible
|
||||
testFileNames = args.testFileNames.split()
|
||||
testFilePaths = []
|
||||
for fileName in testFileNames:
|
||||
fileName = os.path.expanduser(fileName)
|
||||
if os.path.isfile(fileName) or os.path.isdir(fileName):
|
||||
testFilePaths.append(os.path.abspath(fileName))
|
||||
else:
|
||||
log("ERROR: File/directory not found: " + fileName)
|
||||
exit(1)
|
||||
|
||||
# check if dictionary is accessible
|
||||
if args.dictionary:
|
||||
args.dictionary = os.path.abspath(os.path.expanduser(args.dictionary))
|
||||
if not os.path.isfile(args.dictionary):
|
||||
log("ERROR: Dictionary not found: " + args.dictionary)
|
||||
exit(1)
|
||||
|
||||
# check availability of e-mail senders
|
||||
have_mutt = does_command_exist("mutt -h")
|
||||
have_mail = does_command_exist("mail -V")
|
||||
if not have_mutt and not have_mail:
|
||||
log("ERROR: e-mail senders 'mail' or 'mutt' not found")
|
||||
exit(1)
|
||||
|
||||
clang_version = execute("clang -v 2>&1 | grep ' version ' | sed -e 's:.*version \\([0-9.]*\\).*:\\1:' -e 's:\\.\\([0-9][0-9]\\):\\1:g'", verbose)[0];
|
||||
gcc_version = execute("gcc -dumpversion", verbose)[0];
|
||||
|
||||
if verbose:
|
||||
print("PARAMETERS:\nrepoURL=%s" % args.repoURL)
|
||||
print("working_path=%s" % working_path)
|
||||
print("clone_path=%s" % clone_path)
|
||||
print("testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths))
|
||||
print("message=%s" % args.message)
|
||||
print("emails=%s" % args.emails)
|
||||
print("dictionary=%s" % args.dictionary)
|
||||
print("maxLoadAvg=%s" % args.maxLoadAvg)
|
||||
print("lowerLimit=%s" % args.lowerLimit)
|
||||
print("ratioLimit=%s" % args.ratioLimit)
|
||||
print("lastCLevel=%s" % args.lastCLevel)
|
||||
print("sleepTime=%s" % args.sleepTime)
|
||||
print("timeout=%s" % args.timeout)
|
||||
print("dry_run=%s" % args.dry_run)
|
||||
print("verbose=%s" % args.verbose)
|
||||
print("have_mutt=%s have_mail=%s" % (have_mutt, have_mail))
|
||||
|
||||
# clone ZSTD repo if needed
|
||||
if not os.path.isdir(working_path):
|
||||
os.mkdir(working_path)
|
||||
if not os.path.isdir(clone_path):
|
||||
execute.cwd = working_path
|
||||
execute('git clone ' + args.repoURL)
|
||||
if not os.path.isdir(clone_path):
|
||||
log("ERROR: ZSTD clone not found: " + clone_path)
|
||||
exit(1)
|
||||
execute.cwd = clone_path
|
||||
|
||||
# check if speedTest.pid already exists
|
||||
pidfile = "./speedTest.pid"
|
||||
if os.path.isfile(pidfile):
|
||||
log("ERROR: %s already exists, exiting" % pidfile)
|
||||
exit(1)
|
||||
|
||||
send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been started' % (email_header, pid, script_version), args.message, have_mutt, have_mail)
|
||||
with open(pidfile, 'w') as the_file:
|
||||
the_file.write(pid)
|
||||
|
||||
branch = ""
|
||||
commit = ""
|
||||
first_time = True
|
||||
while True:
|
||||
try:
|
||||
if first_time:
|
||||
first_time = False
|
||||
else:
|
||||
time.sleep(args.sleepTime)
|
||||
loadavg = os.getloadavg()[0]
|
||||
if (loadavg <= args.maxLoadAvg):
|
||||
branches = git_get_branches()
|
||||
for branch in branches:
|
||||
commit = execute('git show -s --format=%h ' + branch, verbose)[0]
|
||||
last_commit = update_config_file(branch, commit)
|
||||
if commit == last_commit:
|
||||
log("skipping branch %s: head %s already processed" % (branch, commit))
|
||||
else:
|
||||
log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit))
|
||||
execute('git checkout -- . && git checkout ' + branch)
|
||||
print(git_get_changes(branch, commit, last_commit))
|
||||
test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail)
|
||||
else:
|
||||
log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg))
|
||||
if verbose:
|
||||
log("sleep for %s seconds" % args.sleepTime)
|
||||
except Exception as e:
|
||||
stack = traceback.format_exc()
|
||||
email_topic = '[%s:%s] ERROR in %s:%s' % (email_header, pid, branch, commit)
|
||||
send_email(args.emails, email_topic, stack, have_mutt, have_mail)
|
||||
print(stack)
|
||||
except KeyboardInterrupt:
|
||||
os.unlink(pidfile)
|
||||
send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been stopped' % (email_header, pid, script_version), args.message, have_mutt, have_mail)
|
||||
exit(0)
|
||||
184
build_amd64/_deps/zstd-src/tests/README.md
Normal file
184
build_amd64/_deps/zstd-src/tests/README.md
Normal file
@@ -0,0 +1,184 @@
|
||||
Programs and scripts for automated testing of Zstandard
|
||||
=======================================================
|
||||
|
||||
This directory contains the following programs and scripts:
|
||||
- `datagen` : Synthetic and parametrable data generator, for tests
|
||||
- `fullbench` : Precisely measure speed for each zstd inner functions
|
||||
- `fuzzer` : Test tool, to check zstd integrity on target platform
|
||||
- `paramgrill` : parameter tester for zstd
|
||||
- `test-zstd-speed.py` : script for testing zstd speed difference between commits
|
||||
- `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+)
|
||||
- `zstreamtest` : Fuzzer test tool for zstd streaming API
|
||||
- `legacy` : Test tool to test decoding of legacy zstd frames
|
||||
- `decodecorpus` : Tool to generate valid Zstandard frames, for verifying decoder implementations
|
||||
|
||||
|
||||
#### `test-zstd-versions.py` - script for testing zstd interoperability between versions
|
||||
|
||||
This script creates `versionsTest` directory to which zstd repository is cloned.
|
||||
Then all tagged (released) versions of zstd are compiled.
|
||||
In the following step interoperability between zstd versions is checked.
|
||||
|
||||
#### `automated-benchmarking.py` - script for benchmarking zstd prs to dev
|
||||
|
||||
This script benchmarks facebook:dev and changes from pull requests made to zstd and compares
|
||||
them against facebook:dev to detect regressions. This script currently runs on a dedicated
|
||||
desktop machine for every pull request that is made to the zstd repo but can also
|
||||
be run on any machine via the command line interface.
|
||||
|
||||
There are three modes of usage for this script: fastmode will just run a minimal single
|
||||
build comparison (between facebook:dev and facebook:release), onetime will pull all the current
|
||||
pull requests from the zstd repo and compare facebook:dev to all of them once, continuous
|
||||
will continuously get pull requests from the zstd repo and run benchmarks against facebook:dev.
|
||||
|
||||
```
|
||||
Example usage: python automated_benchmarking.py
|
||||
```
|
||||
|
||||
```
|
||||
usage: automated_benchmarking.py [-h] [--directory DIRECTORY]
|
||||
[--levels LEVELS] [--iterations ITERATIONS]
|
||||
[--emails EMAILS] [--frequency FREQUENCY]
|
||||
[--mode MODE] [--dict DICT]
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
--directory DIRECTORY
|
||||
directory with files to benchmark
|
||||
--levels LEVELS levels to test e.g. ('1,2,3')
|
||||
--iterations ITERATIONS
|
||||
number of benchmark iterations to run
|
||||
--emails EMAILS email addresses of people who will be alerted upon
|
||||
regression. Only for continuous mode
|
||||
--frequency FREQUENCY
|
||||
specifies the number of seconds to wait before each
|
||||
successive check for new PRs in continuous mode
|
||||
--mode MODE 'fastmode', 'onetime', 'current', or 'continuous' (see
|
||||
README.md for details)
|
||||
--dict DICT filename of dictionary to use (when set, this
|
||||
dictionary will be used to compress the files provided
|
||||
inside --directory)
|
||||
```
|
||||
|
||||
#### `test-zstd-speed.py` - script for testing zstd speed difference between commits
|
||||
|
||||
DEPRECATED
|
||||
|
||||
This script creates `speedTest` directory to which zstd repository is cloned.
|
||||
Then it compiles all branches of zstd and performs a speed benchmark for a given list of files (the `testFileNames` parameter).
|
||||
After `sleepTime` (an optional parameter, default 300 seconds) seconds the script checks repository for new commits.
|
||||
If a new commit is found it is compiled and a speed benchmark for this commit is performed.
|
||||
The results of the speed benchmark are compared to the previous results.
|
||||
If compression or decompression speed for one of zstd levels is lower than `lowerLimit` (an optional parameter, default 0.98) the speed benchmark is restarted.
|
||||
If second results are also lower than `lowerLimit` the warning e-mail is sent to recipients from the list (the `emails` parameter).
|
||||
|
||||
Additional remarks:
|
||||
- To be sure that speed results are accurate the script should be run on a "stable" target system with no other jobs running in parallel
|
||||
- Using the script with virtual machines can lead to large variations of speed results
|
||||
- The speed benchmark is not performed until computers' load average is lower than `maxLoadAvg` (an optional parameter, default 0.75)
|
||||
- The script sends e-mails using `mutt`; if `mutt` is not available it sends e-mails without attachments using `mail`; if both are not available it only prints a warning
|
||||
|
||||
|
||||
The example usage with two test files, one e-mail address, and with an additional message:
|
||||
```
|
||||
./test-zstd-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60
|
||||
```
|
||||
|
||||
To run the script in background please use:
|
||||
```
|
||||
nohup ./test-zstd-speed.py testFileNames emails &
|
||||
```
|
||||
|
||||
The full list of parameters:
|
||||
```
|
||||
positional arguments:
|
||||
testFileNames file names list for speed benchmark
|
||||
emails list of e-mail addresses to send warnings
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
--message MESSAGE attach an additional message to e-mail
|
||||
--lowerLimit LOWERLIMIT
|
||||
send email if speed is lower than given limit
|
||||
--maxLoadAvg MAXLOADAVG
|
||||
maximum load average to start testing
|
||||
--lastCLevel LASTCLEVEL
|
||||
last compression level for testing
|
||||
--sleepTime SLEEPTIME
|
||||
frequency of repository checking in seconds
|
||||
```
|
||||
|
||||
#### `decodecorpus` - tool to generate Zstandard frames for decoder testing
|
||||
Command line tool to generate test .zst files.
|
||||
|
||||
This tool will generate .zst files with checksums,
|
||||
as well as optionally output the corresponding correct uncompressed data for
|
||||
extra verification.
|
||||
|
||||
Example:
|
||||
```
|
||||
./decodecorpus -ptestfiles -otestfiles -n10000 -s5
|
||||
```
|
||||
will generate 10,000 sample .zst files using a seed of 5 in the `testfiles` directory,
|
||||
with the zstd checksum field set,
|
||||
as well as the 10,000 original files for more detailed comparison of decompression results.
|
||||
|
||||
```
|
||||
./decodecorpus -t -T1mn
|
||||
```
|
||||
will choose a random seed, and for 1 minute,
|
||||
generate random test frames and ensure that the
|
||||
zstd library correctly decompresses them in both simple and streaming modes.
|
||||
|
||||
#### `paramgrill` - tool for generating compression table parameters and optimizing parameters on file given constraints
|
||||
|
||||
Full list of arguments
|
||||
```
|
||||
-T# : set level 1 speed objective
|
||||
-B# : cut input into blocks of size # (default : single block)
|
||||
-S : benchmarks a single run (example command: -Sl3w10h12)
|
||||
w# - windowLog
|
||||
h# - hashLog
|
||||
c# - chainLog
|
||||
s# - searchLog
|
||||
l# - minMatch
|
||||
t# - targetLength
|
||||
S# - strategy
|
||||
L# - level
|
||||
--zstd= : Single run, parameter selection syntax same as zstdcli with more parameters
|
||||
(Added forceAttachDictionary / fadt)
|
||||
When invoked with --optimize, this represents the sample to exceed.
|
||||
--optimize= : find parameters to maximize compression ratio given parameters
|
||||
Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints
|
||||
cSpeed= : Minimum compression speed
|
||||
dSpeed= : Minimum decompression speed
|
||||
cMem= : Maximum compression memory
|
||||
lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed,
|
||||
stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%)
|
||||
: In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters
|
||||
(Lower value will begin with stronger strategies) (Default 90%)
|
||||
speedRatio= (accepts decimals)
|
||||
: determines value of gains in speed vs gains in ratio
|
||||
when determining overall winner (default 5 (1% ratio = 5% speed)).
|
||||
tries= : Maximum number of random restarts on a single strategy before switching (Default 5)
|
||||
Higher values will make optimizer run longer, more chances to find better solution.
|
||||
memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size.
|
||||
Setting memLog = 0 turns off memoization
|
||||
--display= : specify which parameters are included in the output
|
||||
can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters
|
||||
(Default: display all params available)
|
||||
-P# : generated sample compressibility (when no file is provided)
|
||||
-t# : Caps runtime of operation in seconds (default: 99999 seconds (about 27 hours))
|
||||
-v : Prints Benchmarking output
|
||||
-D : Next argument dictionary file
|
||||
-s : Benchmark all files separately
|
||||
-q : Quiet, repeat for more quiet
|
||||
-q Prints parameters + results whenever a new best is found
|
||||
-qq Only prints parameters whenever a new best is found, prints final parameters + results
|
||||
-qqq Only print final parameters + results
|
||||
-qqqq Only prints final parameter set in the form --zstd=
|
||||
-v : Verbose, cancels quiet, repeat for more volume
|
||||
-v Prints all candidate parameters and results
|
||||
|
||||
```
|
||||
Any inputs afterwards are treated as files to benchmark.
|
||||
326
build_amd64/_deps/zstd-src/tests/automated_benchmarking.py
Normal file
326
build_amd64/_deps/zstd-src/tests/automated_benchmarking.py
Normal file
@@ -0,0 +1,326 @@
|
||||
# ################################################################
|
||||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# You may select, at your option, one of the above-listed licenses.
|
||||
# ##########################################################################
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import pickle as pk
|
||||
import subprocess
|
||||
import urllib.request
|
||||
|
||||
|
||||
GITHUB_API_PR_URL = "https://api.github.com/repos/facebook/zstd/pulls?state=open"
|
||||
GITHUB_URL_TEMPLATE = "https://github.com/{}/zstd"
|
||||
RELEASE_BUILD = {"user": "facebook", "branch": "dev", "hash": None}
|
||||
|
||||
# check to see if there are any new PRs every minute
|
||||
DEFAULT_MAX_API_CALL_FREQUENCY_SEC = 60
|
||||
PREVIOUS_PRS_FILENAME = "prev_prs.pk"
|
||||
|
||||
# Not sure what the threshold for triggering alarms should be
|
||||
# 1% regression sounds like a little too sensitive but the desktop
|
||||
# that I'm running it on is pretty stable so I think this is fine
|
||||
CSPEED_REGRESSION_TOLERANCE = 0.01
|
||||
DSPEED_REGRESSION_TOLERANCE = 0.01
|
||||
|
||||
|
||||
def get_new_open_pr_builds(prev_state=True):
|
||||
prev_prs = None
|
||||
if os.path.exists(PREVIOUS_PRS_FILENAME):
|
||||
with open(PREVIOUS_PRS_FILENAME, "rb") as f:
|
||||
prev_prs = pk.load(f)
|
||||
data = json.loads(urllib.request.urlopen(GITHUB_API_PR_URL).read().decode("utf-8"))
|
||||
prs = {
|
||||
d["url"]: {
|
||||
"user": d["user"]["login"],
|
||||
"branch": d["head"]["ref"],
|
||||
"hash": d["head"]["sha"].strip(),
|
||||
}
|
||||
for d in data
|
||||
}
|
||||
with open(PREVIOUS_PRS_FILENAME, "wb") as f:
|
||||
pk.dump(prs, f)
|
||||
if not prev_state or prev_prs == None:
|
||||
return list(prs.values())
|
||||
return [pr for url, pr in prs.items() if url not in prev_prs or prev_prs[url] != pr]
|
||||
|
||||
|
||||
def get_latest_hashes():
|
||||
tmp = subprocess.run(["git", "log", "-1"], stdout=subprocess.PIPE).stdout.decode(
|
||||
"utf-8"
|
||||
)
|
||||
sha1 = tmp.split("\n")[0].split(" ")[1]
|
||||
tmp = subprocess.run(
|
||||
["git", "show", "{}^1".format(sha1)], stdout=subprocess.PIPE
|
||||
).stdout.decode("utf-8")
|
||||
sha2 = tmp.split("\n")[0].split(" ")[1]
|
||||
tmp = subprocess.run(
|
||||
["git", "show", "{}^2".format(sha1)], stdout=subprocess.PIPE
|
||||
).stdout.decode("utf-8")
|
||||
sha3 = "" if len(tmp) == 0 else tmp.split("\n")[0].split(" ")[1]
|
||||
return [sha1.strip(), sha2.strip(), sha3.strip()]
|
||||
|
||||
|
||||
def get_builds_for_latest_hash():
|
||||
hashes = get_latest_hashes()
|
||||
for b in get_new_open_pr_builds(False):
|
||||
if b["hash"] in hashes:
|
||||
return [b]
|
||||
return []
|
||||
|
||||
|
||||
def clone_and_build(build):
|
||||
if build["user"] != None:
|
||||
github_url = GITHUB_URL_TEMPLATE.format(build["user"])
|
||||
os.system(
|
||||
"""
|
||||
rm -rf zstd-{user}-{sha} &&
|
||||
git clone {github_url} zstd-{user}-{sha} &&
|
||||
cd zstd-{user}-{sha} &&
|
||||
{checkout_command}
|
||||
make -j &&
|
||||
cd ../
|
||||
""".format(
|
||||
user=build["user"],
|
||||
github_url=github_url,
|
||||
sha=build["hash"],
|
||||
checkout_command="git checkout {} &&".format(build["hash"])
|
||||
if build["hash"] != None
|
||||
else "",
|
||||
)
|
||||
)
|
||||
return "zstd-{user}-{sha}/zstd".format(user=build["user"], sha=build["hash"])
|
||||
else:
|
||||
os.system("cd ../ && make -j && cd tests")
|
||||
return "../zstd"
|
||||
|
||||
|
||||
def parse_benchmark_output(output):
|
||||
idx = [i for i, d in enumerate(output) if d == "MB/s"]
|
||||
return [float(output[idx[0] - 1]), float(output[idx[1] - 1])]
|
||||
|
||||
|
||||
def benchmark_single(executable, level, filename):
|
||||
return parse_benchmark_output((
|
||||
subprocess.run(
|
||||
[executable, "-qb{}".format(level), filename], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
)
|
||||
.stdout.decode("utf-8")
|
||||
.split(" ")
|
||||
))
|
||||
|
||||
|
||||
def benchmark_n(executable, level, filename, n):
|
||||
speeds_arr = [benchmark_single(executable, level, filename) for _ in range(n)]
|
||||
cspeed, dspeed = max(b[0] for b in speeds_arr), max(b[1] for b in speeds_arr)
|
||||
print(
|
||||
"Bench (executable={} level={} filename={}, iterations={}):\n\t[cspeed: {} MB/s, dspeed: {} MB/s]".format(
|
||||
os.path.basename(executable),
|
||||
level,
|
||||
os.path.basename(filename),
|
||||
n,
|
||||
cspeed,
|
||||
dspeed,
|
||||
)
|
||||
)
|
||||
return (cspeed, dspeed)
|
||||
|
||||
|
||||
def benchmark(build, filenames, levels, iterations):
|
||||
executable = clone_and_build(build)
|
||||
return [
|
||||
[benchmark_n(executable, l, f, iterations) for f in filenames] for l in levels
|
||||
]
|
||||
|
||||
|
||||
def benchmark_dictionary_single(executable, filenames_directory, dictionary_filename, level, iterations):
|
||||
cspeeds, dspeeds = [], []
|
||||
for _ in range(iterations):
|
||||
output = subprocess.run([executable, "-qb{}".format(level), "-D", dictionary_filename, "-r", filenames_directory], stdout=subprocess.PIPE).stdout.decode("utf-8").split(" ")
|
||||
cspeed, dspeed = parse_benchmark_output(output)
|
||||
cspeeds.append(cspeed)
|
||||
dspeeds.append(dspeed)
|
||||
max_cspeed, max_dspeed = max(cspeeds), max(dspeeds)
|
||||
print(
|
||||
"Bench (executable={} level={} filenames_directory={}, dictionary_filename={}, iterations={}):\n\t[cspeed: {} MB/s, dspeed: {} MB/s]".format(
|
||||
os.path.basename(executable),
|
||||
level,
|
||||
os.path.basename(filenames_directory),
|
||||
os.path.basename(dictionary_filename),
|
||||
iterations,
|
||||
max_cspeed,
|
||||
max_dspeed,
|
||||
)
|
||||
)
|
||||
return (max_cspeed, max_dspeed)
|
||||
|
||||
|
||||
def benchmark_dictionary(build, filenames_directory, dictionary_filename, levels, iterations):
|
||||
executable = clone_and_build(build)
|
||||
return [benchmark_dictionary_single(executable, filenames_directory, dictionary_filename, l, iterations) for l in levels]
|
||||
|
||||
|
||||
def parse_regressions_and_labels(old_cspeed, new_cspeed, old_dspeed, new_dspeed, baseline_build, test_build):
|
||||
cspeed_reg = (old_cspeed - new_cspeed) / old_cspeed
|
||||
dspeed_reg = (old_dspeed - new_dspeed) / old_dspeed
|
||||
baseline_label = "{}:{} ({})".format(
|
||||
baseline_build["user"], baseline_build["branch"], baseline_build["hash"]
|
||||
)
|
||||
test_label = "{}:{} ({})".format(
|
||||
test_build["user"], test_build["branch"], test_build["hash"]
|
||||
)
|
||||
return cspeed_reg, dspeed_reg, baseline_label, test_label
|
||||
|
||||
|
||||
def get_regressions(baseline_build, test_build, iterations, filenames, levels):
|
||||
old = benchmark(baseline_build, filenames, levels, iterations)
|
||||
new = benchmark(test_build, filenames, levels, iterations)
|
||||
regressions = []
|
||||
for j, level in enumerate(levels):
|
||||
for k, filename in enumerate(filenames):
|
||||
old_cspeed, old_dspeed = old[j][k]
|
||||
new_cspeed, new_dspeed = new[j][k]
|
||||
cspeed_reg, dspeed_reg, baseline_label, test_label = parse_regressions_and_labels(
|
||||
old_cspeed, new_cspeed, old_dspeed, new_dspeed, baseline_build, test_build
|
||||
)
|
||||
if cspeed_reg > CSPEED_REGRESSION_TOLERANCE:
|
||||
regressions.append(
|
||||
"[COMPRESSION REGRESSION] (level={} filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format(
|
||||
level,
|
||||
filename,
|
||||
baseline_label,
|
||||
test_label,
|
||||
old_cspeed,
|
||||
new_cspeed,
|
||||
cspeed_reg * 100.0,
|
||||
)
|
||||
)
|
||||
if dspeed_reg > DSPEED_REGRESSION_TOLERANCE:
|
||||
regressions.append(
|
||||
"[DECOMPRESSION REGRESSION] (level={} filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format(
|
||||
level,
|
||||
filename,
|
||||
baseline_label,
|
||||
test_label,
|
||||
old_dspeed,
|
||||
new_dspeed,
|
||||
dspeed_reg * 100.0,
|
||||
)
|
||||
)
|
||||
return regressions
|
||||
|
||||
def get_regressions_dictionary(baseline_build, test_build, filenames_directory, dictionary_filename, levels, iterations):
|
||||
old = benchmark_dictionary(baseline_build, filenames_directory, dictionary_filename, levels, iterations)
|
||||
new = benchmark_dictionary(test_build, filenames_directory, dictionary_filename, levels, iterations)
|
||||
regressions = []
|
||||
for j, level in enumerate(levels):
|
||||
old_cspeed, old_dspeed = old[j]
|
||||
new_cspeed, new_dspeed = new[j]
|
||||
cspeed_reg, dspeed_reg, baesline_label, test_label = parse_regressions_and_labels(
|
||||
old_cspeed, new_cspeed, old_dspeed, new_dspeed, baseline_build, test_build
|
||||
)
|
||||
if cspeed_reg > CSPEED_REGRESSION_TOLERANCE:
|
||||
regressions.append(
|
||||
"[COMPRESSION REGRESSION] (level={} filenames_directory={} dictionary_filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format(
|
||||
level,
|
||||
filenames_directory,
|
||||
dictionary_filename,
|
||||
baseline_label,
|
||||
test_label,
|
||||
old_cspeed,
|
||||
new_cspeed,
|
||||
cspeed_reg * 100.0,
|
||||
)
|
||||
)
|
||||
if dspeed_reg > DSPEED_REGRESSION_TOLERANCE:
|
||||
regressions.append(
|
||||
"[DECOMPRESSION REGRESSION] (level={} filenames_directory={} dictionary_filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format(
|
||||
level,
|
||||
filenames_directory,
|
||||
dictionary_filename,
|
||||
baseline_label,
|
||||
test_label,
|
||||
old_dspeed,
|
||||
new_dspeed,
|
||||
dspeed_reg * 100.0,
|
||||
)
|
||||
)
|
||||
return regressions
|
||||
|
||||
|
||||
def main(filenames, levels, iterations, builds=None, emails=None, continuous=False, frequency=DEFAULT_MAX_API_CALL_FREQUENCY_SEC, dictionary_filename=None):
|
||||
if builds == None:
|
||||
builds = get_new_open_pr_builds()
|
||||
while True:
|
||||
for test_build in builds:
|
||||
if dictionary_filename == None:
|
||||
regressions = get_regressions(
|
||||
RELEASE_BUILD, test_build, iterations, filenames, levels
|
||||
)
|
||||
else:
|
||||
regressions = get_regressions_dictionary(
|
||||
RELEASE_BUILD, test_build, filenames, dictionary_filename, levels, iterations
|
||||
)
|
||||
body = "\n".join(regressions)
|
||||
if len(regressions) > 0:
|
||||
if emails != None:
|
||||
os.system(
|
||||
"""
|
||||
echo "{}" | mutt -s "[zstd regression] caused by new pr" {}
|
||||
""".format(
|
||||
body, emails
|
||||
)
|
||||
)
|
||||
print("Emails sent to {}".format(emails))
|
||||
print(body)
|
||||
if not continuous:
|
||||
break
|
||||
time.sleep(frequency)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument("--directory", help="directory with files to benchmark", default="golden-compression")
|
||||
parser.add_argument("--levels", help="levels to test e.g. ('1,2,3')", default="1")
|
||||
parser.add_argument("--iterations", help="number of benchmark iterations to run", default="1")
|
||||
parser.add_argument("--emails", help="email addresses of people who will be alerted upon regression. Only for continuous mode", default=None)
|
||||
parser.add_argument("--frequency", help="specifies the number of seconds to wait before each successive check for new PRs in continuous mode", default=DEFAULT_MAX_API_CALL_FREQUENCY_SEC)
|
||||
parser.add_argument("--mode", help="'fastmode', 'onetime', 'current', or 'continuous' (see README.md for details)", default="current")
|
||||
parser.add_argument("--dict", help="filename of dictionary to use (when set, this dictionary will be used to compress the files provided inside --directory)", default=None)
|
||||
|
||||
args = parser.parse_args()
|
||||
filenames = args.directory
|
||||
levels = [int(l) for l in args.levels.split(",")]
|
||||
mode = args.mode
|
||||
iterations = int(args.iterations)
|
||||
emails = args.emails
|
||||
frequency = int(args.frequency)
|
||||
dictionary_filename = args.dict
|
||||
|
||||
if dictionary_filename == None:
|
||||
filenames = glob.glob("{}/**".format(filenames))
|
||||
|
||||
if (len(filenames) == 0):
|
||||
print("0 files found")
|
||||
quit()
|
||||
|
||||
if mode == "onetime":
|
||||
main(filenames, levels, iterations, frequency=frequenc, dictionary_filename=dictionary_filename)
|
||||
elif mode == "current":
|
||||
builds = [{"user": None, "branch": "None", "hash": None}]
|
||||
main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename)
|
||||
elif mode == "fastmode":
|
||||
builds = [{"user": "facebook", "branch": "release", "hash": None}]
|
||||
main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename)
|
||||
else:
|
||||
main(filenames, levels, iterations, None, emails, True, frequency=frequency, dictionary_filename=dictionary_filename)
|
||||
133
build_amd64/_deps/zstd-src/tests/bigdict.c
Normal file
133
build_amd64/_deps/zstd-src/tests/bigdict.c
Normal file
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#include <stddef.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdint.h>
|
||||
#include "datagen.h"
|
||||
#include "mem.h"
|
||||
#define ZSTD_STATIC_LINKING_ONLY
|
||||
#include "zstd.h"
|
||||
|
||||
static int
|
||||
compress(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
|
||||
void* dst, size_t dstCapacity,
|
||||
void const* src, size_t srcSize,
|
||||
void* roundtrip, ZSTD_EndDirective end)
|
||||
{
|
||||
ZSTD_inBuffer in = {src, srcSize, 0};
|
||||
ZSTD_outBuffer out = {dst, dstCapacity, 0};
|
||||
int ended = 0;
|
||||
|
||||
while (!ended && (in.pos < in.size || out.pos > 0)) {
|
||||
size_t rc;
|
||||
out.pos = 0;
|
||||
rc = ZSTD_compressStream2(cctx, &out, &in, end);
|
||||
if (ZSTD_isError(rc))
|
||||
return 1;
|
||||
if (end == ZSTD_e_end && rc == 0)
|
||||
ended = 1;
|
||||
{
|
||||
ZSTD_inBuffer rtIn = {dst, out.pos, 0};
|
||||
ZSTD_outBuffer rtOut = {roundtrip, srcSize, 0};
|
||||
rc = 1;
|
||||
while (rtIn.pos < rtIn.size || rtOut.pos > 0) {
|
||||
rtOut.pos = 0;
|
||||
rc = ZSTD_decompressStream(dctx, &rtOut, &rtIn);
|
||||
if (ZSTD_isError(rc)) {
|
||||
fprintf(stderr, "Decompression error: %s\n", ZSTD_getErrorName(rc));
|
||||
return 1;
|
||||
}
|
||||
if (rc == 0)
|
||||
break;
|
||||
}
|
||||
if (ended && rc != 0) {
|
||||
fprintf(stderr, "Frame not finished!\n");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
const size_t dataSize = (size_t)1 << 30;
|
||||
const size_t outSize = ZSTD_compressBound(dataSize);
|
||||
const size_t bufferSize = (size_t)1 << 31;
|
||||
char* buffer = (char*)malloc(bufferSize);
|
||||
void* out = malloc(outSize);
|
||||
void* roundtrip = malloc(dataSize);
|
||||
int _exit_code = 0;
|
||||
(void)argc;
|
||||
(void)argv;
|
||||
|
||||
if (!buffer || !out || !roundtrip || !cctx || !dctx) {
|
||||
fprintf(stderr, "Allocation failure\n");
|
||||
_exit_code = 1;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 31)))
|
||||
return 1;
|
||||
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)))
|
||||
return 1;
|
||||
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_overlapLog, 9)))
|
||||
return 1;
|
||||
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)))
|
||||
return 1;
|
||||
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt)))
|
||||
return 1;
|
||||
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, 7)))
|
||||
return 1;
|
||||
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7)))
|
||||
return 1;
|
||||
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, 1)))
|
||||
return 1;
|
||||
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 10)))
|
||||
return 1;
|
||||
if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, 10)))
|
||||
return 1;
|
||||
|
||||
if (ZSTD_isError(ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 31)))
|
||||
return 1;
|
||||
|
||||
RDG_genBuffer(buffer, bufferSize, 1.0, 0.0, 0xbeefcafe);
|
||||
|
||||
/* Compress 30 GB */
|
||||
{
|
||||
int i;
|
||||
for (i = 0; i < 10; ++i) {
|
||||
fprintf(stderr, "Compressing 1 GB\n");
|
||||
if (compress(cctx, dctx, out, outSize, buffer, dataSize, roundtrip, ZSTD_e_continue))
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
fprintf(stderr, "Compressing 1 GB\n");
|
||||
if (compress(cctx, dctx, out, outSize, buffer, dataSize, roundtrip, ZSTD_e_end))
|
||||
return 1;
|
||||
|
||||
fprintf(stderr, "Success!\n");
|
||||
|
||||
goto cleanup;
|
||||
|
||||
cleanup:
|
||||
free(roundtrip);
|
||||
free(out);
|
||||
free(buffer);
|
||||
ZSTD_freeCCtx(cctx);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
return _exit_code;
|
||||
}
|
||||
65
build_amd64/_deps/zstd-src/tests/checkTag.c
Normal file
65
build_amd64/_deps/zstd-src/tests/checkTag.c
Normal file
@@ -0,0 +1,65 @@
|
||||
/*
|
||||
* Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
/* checkTag : validation tool for libzstd
|
||||
* command :
|
||||
* $ ./checkTag tag
|
||||
* checkTag validates tags of following format : v[0-9].[0-9].[0-9]{any}
|
||||
* The tag is then compared to zstd version number.
|
||||
* They are compatible if first 3 digits are identical.
|
||||
* Anything beyond that is free, and doesn't impact validation.
|
||||
* Example : tag v1.8.1.2 is compatible with version 1.8.1
|
||||
* When tag and version are not compatible, program exits with error code 1.
|
||||
* When they are compatible, it exists with a code 0.
|
||||
* checkTag is intended to be used in automated testing environment.
|
||||
*/
|
||||
|
||||
#include <stdio.h> /* printf */
|
||||
#include <string.h> /* strlen, strncmp */
|
||||
#include "zstd.h" /* ZSTD_VERSION_STRING */
|
||||
|
||||
|
||||
/* validate() :
|
||||
* @return 1 if tag is compatible, 0 if not.
|
||||
*/
|
||||
static int validate(const char* const tag)
|
||||
{
|
||||
size_t const tagLength = strlen(tag);
|
||||
size_t const verLength = strlen(ZSTD_VERSION_STRING);
|
||||
|
||||
if (tagLength < 2) return 0;
|
||||
if (tag[0] != 'v') return 0;
|
||||
if (tagLength <= verLength) return 0;
|
||||
|
||||
if (strncmp(ZSTD_VERSION_STRING, tag+1, verLength)) return 0;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int main(int argc, const char** argv)
|
||||
{
|
||||
const char* const exeName = argv[0];
|
||||
const char* const tag = argv[1];
|
||||
if (argc!=2) {
|
||||
printf("incorrect usage : %s tag \n", exeName);
|
||||
return 2;
|
||||
}
|
||||
|
||||
printf("Version : %s \n", ZSTD_VERSION_STRING);
|
||||
printf("Tag : %s \n", tag);
|
||||
|
||||
if (validate(tag)) {
|
||||
printf("OK : tag is compatible with zstd version \n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
printf("!! error : tag and versions are not compatible !! \n");
|
||||
return 1;
|
||||
}
|
||||
31
build_amd64/_deps/zstd-src/tests/check_size.py
Executable file
31
build_amd64/_deps/zstd-src/tests/check_size.py
Executable file
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
# ################################################################
|
||||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under both the BSD-style license (found in the
|
||||
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
# in the COPYING file in the root directory of this source tree).
|
||||
# You may select, at your option, one of the above-listed licenses.
|
||||
# ################################################################
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
if len(sys.argv) != 3:
|
||||
print(f"Usage: {sys.argv[0]} FILE SIZE_LIMIT")
|
||||
sys.exit(1)
|
||||
|
||||
file = sys.argv[1]
|
||||
limit = int(sys.argv[2])
|
||||
|
||||
if not os.path.exists(file):
|
||||
print(f"{file} does not exist")
|
||||
sys.exit(1)
|
||||
|
||||
size = os.path.getsize(file)
|
||||
|
||||
if size > limit:
|
||||
print(f"file {file} is {size} bytes, which is greater than the limit of {limit} bytes")
|
||||
sys.exit(1)
|
||||
6
build_amd64/_deps/zstd-src/tests/cli-tests/.gitignore
vendored
Normal file
6
build_amd64/_deps/zstd-src/tests/cli-tests/.gitignore
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
!bin/
|
||||
!datagen
|
||||
!zstdcat
|
||||
|
||||
scratch/
|
||||
bin/symlinks
|
||||
258
build_amd64/_deps/zstd-src/tests/cli-tests/README.md
Normal file
258
build_amd64/_deps/zstd-src/tests/cli-tests/README.md
Normal file
@@ -0,0 +1,258 @@
|
||||
# CLI tests
|
||||
|
||||
The CLI tests are focused on testing the zstd CLI.
|
||||
They are intended to be simple tests that the CLI and arguments work as advertised.
|
||||
They are not intended to test the library, only the code in `programs/`.
|
||||
The library will get incidental coverage, but if you find yourself trying to trigger a specific condition in the library, this is the wrong tool.
|
||||
|
||||
## Test runner usage
|
||||
|
||||
The test runner `run.py` will run tests against the in-tree build of `zstd` and `datagen` by default. Which means that `zstd` and `datagen` must be built.
|
||||
|
||||
The `zstd` binary used can be passed with `--zstd /path/to/zstd`.
|
||||
Additionally, to run `zstd` through a tool like `valgrind` or `qemu`, set the `--exec-prefix 'valgrind -q'` flag.
|
||||
|
||||
Similarly, the `--datagen`, and `--zstdgrep` flags can be set to specify
|
||||
the paths to their respective binaries. However, these tools do not use
|
||||
the `EXEC_PREFIX`.
|
||||
|
||||
Each test executes in its own scratch directory under `scratch/test/name`. E.g. `scratch/basic/help.sh/`. Normally these directories are removed after the test executes. However, the `--preserve` flag will preserve these directories after execution, and save the tests exit code, stdout, and stderr in the scratch directory to `exit`, `stderr`, and `stdout` respectively. This can be useful for debugging/editing a test and updating the expected output.
|
||||
|
||||
### Running all the tests
|
||||
|
||||
By default the test runner `run.py` will run all the tests, and report the results.
|
||||
|
||||
Examples:
|
||||
|
||||
```
|
||||
./run.py
|
||||
./run.py --preserve
|
||||
./run.py --zstd ../../build/programs/zstd --datagen ../../build/tests/datagen
|
||||
```
|
||||
|
||||
### Running specific tests
|
||||
|
||||
A set of test names can be passed to the test runner `run.py` to only execute those tests.
|
||||
This can be useful for writing or debugging a test, especially with `--preserve`.
|
||||
|
||||
The test name can either be the path to the test file, or the test name, which is the path relative to the test directory.
|
||||
|
||||
Examples:
|
||||
|
||||
```
|
||||
./run.py basic/help.sh
|
||||
./run.py --preserve basic/help.sh basic/version.sh
|
||||
./run.py --preserve --verbose basic/help.sh
|
||||
```
|
||||
|
||||
### Updating exact output
|
||||
|
||||
If a test is failing because a `.stderr.exact` or `.stdout.exact` no longer matches, you can re-run the tests with `--set-exact-output` and the correct output will be written.
|
||||
|
||||
Example:
|
||||
```
|
||||
./run.py --set-exact-output
|
||||
./run.py basic/help.sh --set-exact-output
|
||||
```
|
||||
|
||||
## Writing a test
|
||||
|
||||
Test cases are arbitrary executables, and can be written in any language, but are generally shell scripts.
|
||||
After the script executes, the exit code, stderr, and stdout are compared against the expectations.
|
||||
|
||||
Each test is run in a clean directory that the test can use for intermediate files. This directory will be cleaned up at the end of the test, unless `--preserve` is passed to the test runner. Additionally, the `setup` script can prepare the directory before the test runs.
|
||||
|
||||
### Calling zstd, utilities, and environment variables
|
||||
|
||||
The `$PATH` for tests is prepended with the `bin/` sub-directory, which contains helper scripts for ease of testing.
|
||||
The `zstd` binary will call the zstd binary specified by `run.py` with the correct `$EXEC_PREFIX`.
|
||||
Similarly, `datagen`, `unzstd`, `zstdgrep`, `zstdcat`, etc, are provided.
|
||||
|
||||
Helper utilities like `cmp_size`, `println`, and `die` are provided here too. See their scripts for details.
|
||||
|
||||
Common shell script libraries are provided under `common/`, with helper variables and functions. They can be sourced with `source "$COMMON/library.sh`.
|
||||
|
||||
Lastly, environment variables are provided for testing, which can be listed when calling `run.py` with `--verbose`.
|
||||
They are generally used by the helper scripts in `bin/` to coordinate everything.
|
||||
|
||||
### Basic test case
|
||||
|
||||
When executing your `$TEST` executable, by default the exit code is expected to be `0`. However, you can provide an alternate expected exit code in a `$TEST.exit` file.
|
||||
|
||||
When executing your `$TEST` executable, by default the expected stderr and stdout are empty. However, you can override the default by providing one of three files:
|
||||
|
||||
* `$TEST.{stdout,stderr}.exact`
|
||||
* `$TEST.{stdout,stderr}.glob`
|
||||
* `$TEST.{stdout,stderr}.ignore`
|
||||
|
||||
If you provide a `.exact` file, the output is expected to exactly match, byte-for-byte.
|
||||
|
||||
If you provide a `.glob` file, the output is expected to match the expected file, where each line is interpreted as a glob syntax. Additionally, a line containing only `...` matches all lines until the next expected line matches.
|
||||
|
||||
If you provide a `.ignore` file, the output is ignored.
|
||||
|
||||
#### Passing examples
|
||||
|
||||
All these examples pass.
|
||||
|
||||
Exit 1, and change the expectation to be 1.
|
||||
|
||||
```
|
||||
exit-1.sh
|
||||
---
|
||||
#!/bin/sh
|
||||
exit 1
|
||||
---
|
||||
|
||||
exit-1.sh.exit
|
||||
---
|
||||
1
|
||||
---
|
||||
```
|
||||
|
||||
Check the stdout output exactly matches.
|
||||
|
||||
```
|
||||
echo.sh
|
||||
---
|
||||
#!/bin/sh
|
||||
echo "hello world"
|
||||
---
|
||||
|
||||
echo.sh.stdout.exact
|
||||
---
|
||||
hello world
|
||||
---
|
||||
```
|
||||
|
||||
Check the stderr output using a glob.
|
||||
|
||||
```
|
||||
random.sh
|
||||
---
|
||||
#!/bin/sh
|
||||
head -c 10 < /dev/urandom | xxd >&2
|
||||
---
|
||||
|
||||
random.sh.stderr.glob
|
||||
---
|
||||
00000000: * * * * * *
|
||||
```
|
||||
|
||||
Multiple lines can be matched with ...
|
||||
|
||||
```
|
||||
random-num-lines.sh
|
||||
---
|
||||
#!/bin/sh
|
||||
echo hello
|
||||
seq 0 $RANDOM
|
||||
echo world
|
||||
---
|
||||
|
||||
random-num-lines.sh.stdout.glob
|
||||
---
|
||||
hello
|
||||
0
|
||||
...
|
||||
world
|
||||
---
|
||||
```
|
||||
|
||||
#### Failing examples
|
||||
|
||||
Exit code is expected to be 0, but is 1.
|
||||
|
||||
```
|
||||
exit-1.sh
|
||||
---
|
||||
#!/bin/sh
|
||||
exit 1
|
||||
---
|
||||
```
|
||||
|
||||
Stdout is expected to be empty, but isn't.
|
||||
|
||||
```
|
||||
echo.sh
|
||||
---
|
||||
#!/bin/sh
|
||||
echo hello world
|
||||
```
|
||||
|
||||
Stderr is expected to be hello but is world.
|
||||
|
||||
```
|
||||
hello.sh
|
||||
---
|
||||
#!/bin/sh
|
||||
echo world >&2
|
||||
---
|
||||
|
||||
hello.sh.stderr.exact
|
||||
---
|
||||
hello
|
||||
---
|
||||
```
|
||||
|
||||
### Setup & teardown scripts
|
||||
|
||||
Finally, test writing can be eased with setup and teardown scripts.
|
||||
Each directory in the test directory is a test-suite consisting of all tests within that directory (but not sub-directories).
|
||||
This test suite can come with 4 scripts to help test writing:
|
||||
|
||||
* `setup_once`
|
||||
* `teardown_once`
|
||||
* `setup`
|
||||
* `teardown`
|
||||
|
||||
The `setup_once` and `teardown_once` are run once before and after all the tests in the suite respectively.
|
||||
They operate in the scratch directory for the test suite, which is the parent directory of each scratch directory for each test case.
|
||||
They can do work that is shared between tests to improve test efficiency.
|
||||
For example, the `dictionaries/setup_once` script builds several dictionaries, for use in the `dictionaries` tests.
|
||||
|
||||
The `setup` and `teardown` scripts run before and after each test case respectively, in the test case's scratch directory.
|
||||
These scripts can do work that is shared between test cases to make tests more succinct.
|
||||
For example, the `dictionaries/setup` script copies the dictionaries built by the `dictionaries/setup_once` script into the test's scratch directory, to make them easier to use, and make sure they aren't accidentally modified.
|
||||
|
||||
#### Examples
|
||||
|
||||
```
|
||||
basic/setup
|
||||
---
|
||||
#!/bin/sh
|
||||
# Create some files for testing with
|
||||
datagen > file
|
||||
datagen > file0
|
||||
datagen > file1
|
||||
---
|
||||
|
||||
basic/test.sh
|
||||
---
|
||||
#!/bin/sh
|
||||
zstd file file0 file1
|
||||
---
|
||||
|
||||
dictionaries/setup_once
|
||||
---
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
mkdir files/ dicts/
|
||||
for i in $(seq 10); do
|
||||
datagen -g1000 > files/$i
|
||||
done
|
||||
|
||||
zstd --train -r files/ -o dicts/0
|
||||
---
|
||||
|
||||
dictionaries/setup
|
||||
---
|
||||
#!/bin/sh
|
||||
|
||||
# Runs in the test case's scratch directory.
|
||||
# The test suite's scratch directory that
|
||||
# `setup_once` operates in is the parent directory.
|
||||
cp -r ../files ../dicts .
|
||||
---
|
||||
```
|
||||
10
build_amd64/_deps/zstd-src/tests/cli-tests/basic/args.sh
Executable file
10
build_amd64/_deps/zstd-src/tests/cli-tests/basic/args.sh
Executable file
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
|
||||
println "+ zstd --blah" >&2
|
||||
zstd --blah
|
||||
println "+ zstd -xz" >&2
|
||||
zstd -xz
|
||||
println "+ zstd --adapt=min=1,maxx=2 file.txt" >&2
|
||||
zstd --adapt=min=1,maxx=2 file.txt
|
||||
println "+ zstd --train-cover=k=48,d=8,steps32 file.txt" >&2
|
||||
zstd --train-cover=k=48,d=8,steps32 file.txt
|
||||
@@ -0,0 +1 @@
|
||||
1
|
||||
@@ -0,0 +1,28 @@
|
||||
+ zstd --blah
|
||||
Incorrect parameter: --blah
|
||||
...
|
||||
Usage: zstd *
|
||||
|
||||
Options:
|
||||
...
|
||||
+ zstd -xz
|
||||
Incorrect parameter: -x
|
||||
...
|
||||
Usage: zstd *
|
||||
|
||||
Options:
|
||||
...
|
||||
+ zstd --adapt=min=1,maxx=2 file.txt
|
||||
Incorrect parameter: --adapt=min=1,maxx=2
|
||||
...
|
||||
Usage: zstd *
|
||||
|
||||
Options:
|
||||
...
|
||||
+ zstd --train-cover=k=48,d=8,steps32 file.txt
|
||||
Incorrect parameter: --train-cover=k=48,d=8,steps32
|
||||
...
|
||||
Usage: zstd *
|
||||
|
||||
Options:
|
||||
...
|
||||
10
build_amd64/_deps/zstd-src/tests/cli-tests/basic/help.sh
Executable file
10
build_amd64/_deps/zstd-src/tests/cli-tests/basic/help.sh
Executable file
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
println "+ zstd -h"
|
||||
zstd -h
|
||||
println "+ zstd -H"
|
||||
zstd -H
|
||||
println "+ zstd --help"
|
||||
zstd --help
|
||||
@@ -0,0 +1,34 @@
|
||||
+ zstd -h
|
||||
Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided.
|
||||
|
||||
Usage: zstd *OPTIONS...* *INPUT... | -* *-o OUTPUT*
|
||||
|
||||
Options:
|
||||
-o OUTPUT Write output to a single file, OUTPUT.
|
||||
-k, --keep Preserve INPUT file(s). *Default*
|
||||
--rm Remove INPUT file(s) after successful (de)compression.
|
||||
|
||||
-# Desired compression level, where `#` is a number between 1 and 19;
|
||||
lower numbers provide faster compression, higher numbers yield
|
||||
better compression ratios. *Default: 3*
|
||||
|
||||
-d, --decompress Perform decompression.
|
||||
-D DICT Use DICT as the dictionary for compression or decompression.
|
||||
|
||||
-f, --force Disable input and output checks. Allows overwriting existing files,
|
||||
receiving input from the console, printing output to STDOUT, and
|
||||
operating on links, block devices, etc. Unrecognized formats will be
|
||||
passed-through through as-is.
|
||||
|
||||
-h Display short usage and exit.
|
||||
-H, --help Display full help and exit.
|
||||
-V, --version Display the program version and exit.
|
||||
|
||||
+ zstd -H
|
||||
...
|
||||
Advanced options:
|
||||
...
|
||||
+ zstd --help
|
||||
...
|
||||
Advanced options:
|
||||
...
|
||||
40
build_amd64/_deps/zstd-src/tests/cli-tests/basic/memlimit.sh
Executable file
40
build_amd64/_deps/zstd-src/tests/cli-tests/basic/memlimit.sh
Executable file
@@ -0,0 +1,40 @@
|
||||
#!/bin/sh
|
||||
|
||||
echo "some data" > file
|
||||
|
||||
println "+ zstd --memory=32LB file"
|
||||
zstd --memory=32LB file && die "Should not allow bogus suffix"
|
||||
println "+ zstd --memory=32LiB file"
|
||||
zstd --memory=32LiB file && die "Should not allow bogus suffix"
|
||||
println "+ zstd --memory=32A file"
|
||||
zstd --memory=32A file && die "Should not allow bogus suffix"
|
||||
println "+ zstd --memory=32r82347dn83 file"
|
||||
zstd --memory=32r82347dn83 file && die "Should not allow bogus suffix"
|
||||
println "+ zstd --memory=32asbdf file"
|
||||
zstd --memory=32asbdf file && die "Should not allow bogus suffix"
|
||||
println "+ zstd --memory=hello file"
|
||||
zstd --memory=hello file && die "Should not allow non-numeric parameter"
|
||||
println "+ zstd --memory=1 file"
|
||||
zstd -q --memory=1 file && die "Should allow numeric parameter without suffix"
|
||||
rm file.zst
|
||||
println "+ zstd --memory=1K file"
|
||||
zstd -q --memory=1K file && die "Should allow numeric parameter with expected suffix"
|
||||
rm file.zst
|
||||
println "+ zstd --memory=1KB file"
|
||||
zstd -q --memory=1KB file && die "Should allow numeric parameter with expected suffix"
|
||||
rm file.zst
|
||||
println "+ zstd --memory=1KiB file"
|
||||
zstd -q --memory=1KiB file && die "Should allow numeric parameter with expected suffix"
|
||||
rm file.zst
|
||||
println "+ zstd --memory=1M file"
|
||||
zstd -q --memory=1M file && die "Should allow numeric parameter with expected suffix"
|
||||
rm file.zst
|
||||
println "+ zstd --memory=1MB file"
|
||||
zstd -q --memory=1MB file && die "Should allow numeric parameter with expected suffix"
|
||||
rm file.zst
|
||||
println "+ zstd --memory=1MiB file"
|
||||
zstd -q --memory=1MiB file && die "Should allow numeric parameter with expected suffix"
|
||||
rm file.zst
|
||||
|
||||
rm file
|
||||
exit 0
|
||||
@@ -0,0 +1,13 @@
|
||||
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
|
||||
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
|
||||
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
|
||||
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
|
||||
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
|
||||
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
|
||||
Should allow numeric parameter without suffix
|
||||
Should allow numeric parameter with expected suffix
|
||||
Should allow numeric parameter with expected suffix
|
||||
Should allow numeric parameter with expected suffix
|
||||
Should allow numeric parameter with expected suffix
|
||||
Should allow numeric parameter with expected suffix
|
||||
Should allow numeric parameter with expected suffix
|
||||
@@ -0,0 +1,13 @@
|
||||
+ zstd --memory=32LB file
|
||||
+ zstd --memory=32LiB file
|
||||
+ zstd --memory=32A file
|
||||
+ zstd --memory=32r82347dn83 file
|
||||
+ zstd --memory=32asbdf file
|
||||
+ zstd --memory=hello file
|
||||
+ zstd --memory=1 file
|
||||
+ zstd --memory=1K file
|
||||
+ zstd --memory=1KB file
|
||||
+ zstd --memory=1KiB file
|
||||
+ zstd --memory=1M file
|
||||
+ zstd --memory=1MB file
|
||||
+ zstd --memory=1MiB file
|
||||
7
build_amd64/_deps/zstd-src/tests/cli-tests/basic/output_dir.sh
Executable file
7
build_amd64/_deps/zstd-src/tests/cli-tests/basic/output_dir.sh
Executable file
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
|
||||
println "+ zstd -r * --output-dir-mirror=\"\""
|
||||
zstd -r * --output-dir-mirror="" && die "Should not allow empty output dir!"
|
||||
println "+ zstd -r * --output-dir-flat=\"\""
|
||||
zstd -r * --output-dir-flat="" && die "Should not allow empty output dir!"
|
||||
exit 0
|
||||
@@ -0,0 +1,2 @@
|
||||
error: output dir cannot be empty string (did you mean to pass '.' instead?)
|
||||
error: output dir cannot be empty string (did you mean to pass '.' instead?)
|
||||
@@ -0,0 +1,2 @@
|
||||
+ zstd -r * --output-dir-mirror=""
|
||||
+ zstd -r * --output-dir-flat=""
|
||||
6
build_amd64/_deps/zstd-src/tests/cli-tests/basic/version.sh
Executable file
6
build_amd64/_deps/zstd-src/tests/cli-tests/basic/version.sh
Executable file
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
zstd -V
|
||||
zstd --version
|
||||
@@ -0,0 +1,2 @@
|
||||
*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***
|
||||
*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***
|
||||
44
build_amd64/_deps/zstd-src/tests/cli-tests/bin/cmp_size
Executable file
44
build_amd64/_deps/zstd-src/tests/cli-tests/bin/cmp_size
Executable file
@@ -0,0 +1,44 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
usage()
|
||||
{
|
||||
printf "USAGE:\n\t$0 [-eq|-ne|-lt|-le|-gt|-ge] FILE1 FILE2\n"
|
||||
}
|
||||
|
||||
help()
|
||||
{
|
||||
printf "Small utility to compare file sizes without printing them with set -x.\n\n"
|
||||
usage
|
||||
}
|
||||
|
||||
case "$1" in
|
||||
-h) help; exit 0 ;;
|
||||
--help) help; exit 0 ;;
|
||||
esac
|
||||
|
||||
if ! test -f $2; then
|
||||
printf "FILE1='%b' is not a file\n\n" "$2"
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! test -f $3; then
|
||||
printf "FILE2='%b' is not a file\n\n" "$3"
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
size1=$(wc -c < $2)
|
||||
size2=$(wc -c < $3)
|
||||
|
||||
case "$1" in
|
||||
-eq) [ "$size1" -eq "$size2" ] ;;
|
||||
-ne) [ "$size1" -ne "$size2" ] ;;
|
||||
-lt) [ "$size1" -lt "$size2" ] ;;
|
||||
-le) [ "$size1" -le "$size2" ] ;;
|
||||
-gt) [ "$size1" -gt "$size2" ] ;;
|
||||
-ge) [ "$size1" -ge "$size2" ] ;;
|
||||
esac
|
||||
3
build_amd64/_deps/zstd-src/tests/cli-tests/bin/datagen
Executable file
3
build_amd64/_deps/zstd-src/tests/cli-tests/bin/datagen
Executable file
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
|
||||
"$DATAGEN_BIN" $@
|
||||
4
build_amd64/_deps/zstd-src/tests/cli-tests/bin/die
Executable file
4
build_amd64/_deps/zstd-src/tests/cli-tests/bin/die
Executable file
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
|
||||
println "${*}" 1>&2
|
||||
exit 1
|
||||
2
build_amd64/_deps/zstd-src/tests/cli-tests/bin/println
Executable file
2
build_amd64/_deps/zstd-src/tests/cli-tests/bin/println
Executable file
@@ -0,0 +1,2 @@
|
||||
#!/bin/sh
|
||||
printf '%b\n' "${*}"
|
||||
1
build_amd64/_deps/zstd-src/tests/cli-tests/bin/unzstd
Symbolic link
1
build_amd64/_deps/zstd-src/tests/cli-tests/bin/unzstd
Symbolic link
@@ -0,0 +1 @@
|
||||
zstd
|
||||
9
build_amd64/_deps/zstd-src/tests/cli-tests/bin/zstd
Executable file
9
build_amd64/_deps/zstd-src/tests/cli-tests/bin/zstd
Executable file
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
|
||||
zstdname=$(basename $0)
|
||||
|
||||
if [ -z "$EXEC_PREFIX" ]; then
|
||||
"$ZSTD_SYMLINK_DIR/$zstdname" $@
|
||||
else
|
||||
$EXEC_PREFIX "$ZSTD_SYMLINK_DIR/$zstdname" $@
|
||||
fi
|
||||
1
build_amd64/_deps/zstd-src/tests/cli-tests/bin/zstdcat
Symbolic link
1
build_amd64/_deps/zstd-src/tests/cli-tests/bin/zstdcat
Symbolic link
@@ -0,0 +1 @@
|
||||
zstd
|
||||
2
build_amd64/_deps/zstd-src/tests/cli-tests/bin/zstdgrep
Executable file
2
build_amd64/_deps/zstd-src/tests/cli-tests/bin/zstdgrep
Executable file
@@ -0,0 +1,2 @@
|
||||
#!/bin/sh
|
||||
"$ZSTDGREP_BIN" $@
|
||||
2
build_amd64/_deps/zstd-src/tests/cli-tests/bin/zstdless
Executable file
2
build_amd64/_deps/zstd-src/tests/cli-tests/bin/zstdless
Executable file
@@ -0,0 +1,2 @@
|
||||
#!/bin/sh
|
||||
"$ZSTDLESS_BIN" $@
|
||||
6
build_amd64/_deps/zstd-src/tests/cli-tests/cltools/setup
Executable file
6
build_amd64/_deps/zstd-src/tests/cli-tests/cltools/setup
Executable file
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
echo "1234" > file
|
||||
zstd file
|
||||
8
build_amd64/_deps/zstd-src/tests/cli-tests/cltools/zstdgrep.sh
Executable file
8
build_amd64/_deps/zstd-src/tests/cli-tests/cltools/zstdgrep.sh
Executable file
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
println "+ good path"
|
||||
zstdgrep "1234" file file.zst
|
||||
println "+ bad path"
|
||||
zstdgrep "1234" bad.zst
|
||||
@@ -0,0 +1 @@
|
||||
1
|
||||
@@ -0,0 +1 @@
|
||||
zstd: can't stat bad.zst : No such file or directory -- ignored
|
||||
@@ -0,0 +1,4 @@
|
||||
+ good path
|
||||
file:1234
|
||||
file.zst:1234
|
||||
+ bad path
|
||||
10
build_amd64/_deps/zstd-src/tests/cli-tests/cltools/zstdless.sh
Executable file
10
build_amd64/_deps/zstd-src/tests/cli-tests/cltools/zstdless.sh
Executable file
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
println "+ good path"
|
||||
zstdless file.zst
|
||||
println "+ pass parameters"
|
||||
zstdless -N file.zst # This parameter does not produce line #s when piped, but still serves to test that the flag went to less and not zstd
|
||||
println "+ bad path"
|
||||
zstdless bad.zst >&2
|
||||
@@ -0,0 +1,2 @@
|
||||
zstd: can't stat bad.zst : No such file or directory -- ignored
|
||||
bad.zst: No such file or directory
|
||||
@@ -0,0 +1,5 @@
|
||||
+ good path
|
||||
1234
|
||||
+ pass parameters
|
||||
1234
|
||||
+ bad path
|
||||
19
build_amd64/_deps/zstd-src/tests/cli-tests/common/format.sh
Normal file
19
build_amd64/_deps/zstd-src/tests/cli-tests/common/format.sh
Normal file
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
|
||||
. "$COMMON/platform.sh"
|
||||
|
||||
zstd_supports_format()
|
||||
{
|
||||
zstd -h | grep > $INTOVOID -- "--format=$1"
|
||||
}
|
||||
|
||||
format_extension()
|
||||
{
|
||||
if [ "$1" = "zstd" ]; then
|
||||
printf "zst"
|
||||
elif [ "$1" = "gzip" ]; then
|
||||
printf "gz"
|
||||
else
|
||||
printf "$1"
|
||||
fi
|
||||
}
|
||||
13
build_amd64/_deps/zstd-src/tests/cli-tests/common/mtime.sh
Normal file
13
build_amd64/_deps/zstd-src/tests/cli-tests/common/mtime.sh
Normal file
@@ -0,0 +1,13 @@
|
||||
. "$COMMON/platform.sh"
|
||||
|
||||
MTIME="stat -c %Y"
|
||||
case "$UNAME" in
|
||||
Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;;
|
||||
esac
|
||||
|
||||
assertSameMTime() {
|
||||
MT1=$($MTIME "$1")
|
||||
MT2=$($MTIME "$2")
|
||||
echo MTIME $MT1 $MT2
|
||||
[ "$MT1" = "$MT2" ] || die "mtime on $1 doesn't match mtime on $2 ($MT1 != $MT2)"
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
. "$COMMON/platform.sh"
|
||||
|
||||
GET_PERMS="stat -c %a"
|
||||
case "$UNAME" in
|
||||
Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;;
|
||||
esac
|
||||
|
||||
assertFilePermissions() {
|
||||
STAT1=$($GET_PERMS "$1")
|
||||
STAT2=$2
|
||||
[ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)"
|
||||
}
|
||||
|
||||
assertSamePermissions() {
|
||||
STAT1=$($GET_PERMS "$1")
|
||||
STAT2=$($GET_PERMS "$2")
|
||||
[ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)"
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/bin/sh
|
||||
|
||||
UNAME=$(uname)
|
||||
|
||||
isWindows=false
|
||||
INTOVOID="/dev/null"
|
||||
case "$UNAME" in
|
||||
GNU) DEVDEVICE="/dev/random" ;;
|
||||
*) DEVDEVICE="/dev/zero" ;;
|
||||
esac
|
||||
case "$OS" in
|
||||
Windows*)
|
||||
isWindows=true
|
||||
INTOVOID="NUL"
|
||||
DEVDEVICE="NUL"
|
||||
;;
|
||||
esac
|
||||
|
||||
case "$UNAME" in
|
||||
Darwin) MD5SUM="md5 -r" ;;
|
||||
NetBSD) MD5SUM="md5 -n" ;;
|
||||
OpenBSD) MD5SUM="md5" ;;
|
||||
*) MD5SUM="md5sum" ;;
|
||||
esac
|
||||
|
||||
DIFF="diff"
|
||||
case "$UNAME" in
|
||||
SunOS) DIFF="gdiff" ;;
|
||||
esac
|
||||
|
||||
if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled'
|
||||
then
|
||||
hasMT=""
|
||||
else
|
||||
hasMT="true"
|
||||
fi
|
||||
14
build_amd64/_deps/zstd-src/tests/cli-tests/compression/adapt.sh
Executable file
14
build_amd64/_deps/zstd-src/tests/cli-tests/compression/adapt.sh
Executable file
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
# Test --adapt
|
||||
zstd -f file --adapt -c | zstd -t
|
||||
|
||||
datagen -g100M > file100M
|
||||
|
||||
# Pick parameters to force fast adaptation, even on slow systems
|
||||
zstd --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression"
|
||||
|
||||
# Adaption still happens with --no-progress
|
||||
zstd --no-progress --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression"
|
||||
36
build_amd64/_deps/zstd-src/tests/cli-tests/compression/basic.sh
Executable file
36
build_amd64/_deps/zstd-src/tests/cli-tests/compression/basic.sh
Executable file
@@ -0,0 +1,36 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
# Uncomment the set -v line for debugging
|
||||
# set -v
|
||||
|
||||
# Test compression flags and check that they work
|
||||
zstd file ; zstd -t file.zst
|
||||
zstd -f file ; zstd -t file.zst
|
||||
zstd -f -z file ; zstd -t file.zst
|
||||
zstd -f -k file ; zstd -t file.zst
|
||||
zstd -f -C file ; zstd -t file.zst
|
||||
zstd -f --check file ; zstd -t file.zst
|
||||
zstd -f --no-check file ; zstd -t file.zst
|
||||
zstd -f -- file ; zstd -t file.zst
|
||||
|
||||
# Test output file compression
|
||||
zstd -o file-out.zst ; zstd -t file-out.zst
|
||||
zstd -fo file-out.zst; zstd -t file-out.zst
|
||||
|
||||
# Test compression to stdout
|
||||
zstd -c file | zstd -t
|
||||
zstd --stdout file | zstd -t
|
||||
println bob | zstd | zstd -t
|
||||
|
||||
# Test keeping input file when compressing to stdout in gzip mode
|
||||
if $(command -v $ZSTD_SYMLINK_DIR/gzip); then
|
||||
$ZSTD_SYMLINK_DIR/gzip -c file | zstd -t ; test -f file
|
||||
$ZSTD_SYMLINK_DIR/gzip --stdout file | zstd -t ; test -f file
|
||||
fi
|
||||
|
||||
# Test --rm
|
||||
cp file file-rm
|
||||
zstd --rm file-rm; zstd -t file-rm.zst
|
||||
test ! -f file-rm
|
||||
10
build_amd64/_deps/zstd-src/tests/cli-tests/compression/compress-literals.sh
Executable file
10
build_amd64/_deps/zstd-src/tests/cli-tests/compression/compress-literals.sh
Executable file
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
# Test --[no-]compress-literals
|
||||
zstd file --no-compress-literals -1 -c | zstd -t
|
||||
zstd file --no-compress-literals -19 -c | zstd -t
|
||||
zstd file --no-compress-literals --fast=1 -c | zstd -t
|
||||
zstd file --compress-literals -1 -c | zstd -t
|
||||
zstd file --compress-literals --fast=1 -c | zstd -t
|
||||
16
build_amd64/_deps/zstd-src/tests/cli-tests/compression/format.sh
Executable file
16
build_amd64/_deps/zstd-src/tests/cli-tests/compression/format.sh
Executable file
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
|
||||
. "$COMMON/format.sh"
|
||||
|
||||
set -e
|
||||
|
||||
# Test --format
|
||||
zstd --format=zstd file -f
|
||||
zstd -t file.zst
|
||||
for format in "gzip" "lz4" "xz" "lzma"; do
|
||||
if zstd_supports_format $format; then
|
||||
zstd --format=$format file
|
||||
zstd -t file.$(format_extension $format)
|
||||
zstd -c --format=$format file | zstd -t --format=$format
|
||||
fi
|
||||
done
|
||||
16
build_amd64/_deps/zstd-src/tests/cli-tests/compression/golden.sh
Executable file
16
build_amd64/_deps/zstd-src/tests/cli-tests/compression/golden.sh
Executable file
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-compression/"
|
||||
cp -r "$GOLDEN_DIR" golden/
|
||||
|
||||
zstd -rf golden/ --output-dir-mirror golden-compressed/
|
||||
zstd -r -t golden-compressed/
|
||||
|
||||
zstd --target-compressed-block-size=1024 -rf golden/ --output-dir-mirror golden-compressed/
|
||||
zstd -r -t golden-compressed/
|
||||
|
||||
# PR #3517 block splitter corruption test
|
||||
zstd -rf -19 --zstd=mml=7 golden/ --output-dir-mirror golden-compressed/
|
||||
zstd -r -t golden-compressed/
|
||||
17
build_amd64/_deps/zstd-src/tests/cli-tests/compression/gzip-compat.sh
Executable file
17
build_amd64/_deps/zstd-src/tests/cli-tests/compression/gzip-compat.sh
Executable file
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
# Uncomment the set -v line for debugging
|
||||
# set -v
|
||||
|
||||
# Test gzip specific compression option
|
||||
if $(command -v $ZSTD_SYMLINK_DIR/gzip); then
|
||||
$ZSTD_SYMLINK_DIR/gzip --fast file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
|
||||
$ZSTD_SYMLINK_DIR/gzip --best file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
|
||||
|
||||
# Test -n / --no-name: do not embed original filename in archive
|
||||
$ZSTD_SYMLINK_DIR/gzip -n file ; grep -qv file file.gz ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
|
||||
$ZSTD_SYMLINK_DIR/gzip --no-name file ; grep -qv file file.gz ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
|
||||
$ZSTD_SYMLINK_DIR/gzip -c --no-name file | grep -qv file
|
||||
fi
|
||||
75
build_amd64/_deps/zstd-src/tests/cli-tests/compression/levels.sh
Executable file
75
build_amd64/_deps/zstd-src/tests/cli-tests/compression/levels.sh
Executable file
@@ -0,0 +1,75 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
set -v
|
||||
|
||||
datagen > file
|
||||
|
||||
# Retrieve the program's version information
|
||||
# Note: command echoing differs between macos and linux, so it's disabled below
|
||||
set +v
|
||||
version_info=$(zstd -V)
|
||||
set -v
|
||||
|
||||
# Compress with various levels and ensure that their sizes are ordered
|
||||
zstd --fast=10 file -o file-f10.zst -q
|
||||
zstd --fast=1 file -o file-f1.zst -q
|
||||
zstd -1 file -o file-1.zst -q
|
||||
zstd -19 file -o file-19.zst -q
|
||||
if echo "$version_info" | grep -q '32-bit'; then
|
||||
# skip --max test: not enough address space
|
||||
cp file-19.zst file-max.zst
|
||||
else
|
||||
zstd --max file -o file-max.zst -q
|
||||
fi
|
||||
|
||||
zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-max.zst
|
||||
|
||||
cmp_size -le file-max.zst file-19.zst
|
||||
cmp_size -lt file-19.zst file-1.zst
|
||||
cmp_size -lt file-1.zst file-f1.zst
|
||||
cmp_size -lt file-f1.zst file-f10.zst
|
||||
|
||||
# Test default levels
|
||||
zstd --fast file -f -q
|
||||
cmp file.zst file-f1.zst || die "--fast is not level -1"
|
||||
|
||||
zstd -0 file -o file-0.zst -q
|
||||
zstd -f file -q
|
||||
cmp file.zst file-0.zst || die "Level 0 is not the default level"
|
||||
|
||||
# Test level clamping
|
||||
zstd -99 file -o file-99.zst -q
|
||||
cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19"
|
||||
zstd --fast=200000 file -c | zstd -t
|
||||
|
||||
zstd -5000000000 -f file && die "Level too large, must fail" ||:
|
||||
zstd --fast=5000000000 -f file && die "Level too large, must fail" ||:
|
||||
|
||||
# Test setting a level through the environment variable
|
||||
ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -q
|
||||
ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -q
|
||||
ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -q
|
||||
ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst -q
|
||||
|
||||
cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
|
||||
cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
|
||||
cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level"
|
||||
cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
|
||||
|
||||
# Test invalid environment clevel is the default level
|
||||
zstd -f file -q
|
||||
ZSTD_CLEVEL=- zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
|
||||
ZSTD_CLEVEL=+ zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
|
||||
ZSTD_CLEVEL=a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
|
||||
ZSTD_CLEVEL=-a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
|
||||
ZSTD_CLEVEL=+a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
|
||||
ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
|
||||
ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
|
||||
|
||||
# Test environment clevel is overridden by command line
|
||||
ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -q
|
||||
ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst -q
|
||||
|
||||
cmp file-1.zst file-1-env.zst || die "Environment variable not overridden"
|
||||
cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"
|
||||
@@ -0,0 +1,80 @@
|
||||
|
||||
datagen > file
|
||||
|
||||
# Retrieve the program's version information
|
||||
# Note: command echoing differs between macos and linux, so it's disabled below
|
||||
set +v
|
||||
|
||||
# Compress with various levels and ensure that their sizes are ordered
|
||||
zstd --fast=10 file -o file-f10.zst -q
|
||||
zstd --fast=1 file -o file-f1.zst -q
|
||||
zstd -1 file -o file-1.zst -q
|
||||
zstd -19 file -o file-19.zst -q
|
||||
if echo "$version_info" | grep -q '32-bit'; then
|
||||
# skip --max test: not enough address space
|
||||
cp file-19.zst file-max.zst
|
||||
else
|
||||
zstd --max file -o file-max.zst -q
|
||||
fi
|
||||
|
||||
zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-max.zst
|
||||
5 files decompressed : 327685 bytes total
|
||||
|
||||
cmp_size -le file-max.zst file-19.zst
|
||||
cmp_size -lt file-19.zst file-1.zst
|
||||
cmp_size -lt file-1.zst file-f1.zst
|
||||
cmp_size -lt file-f1.zst file-f10.zst
|
||||
|
||||
# Test default levels
|
||||
zstd --fast file -f -q
|
||||
cmp file.zst file-f1.zst || die "--fast is not level -1"
|
||||
|
||||
zstd -0 file -o file-0.zst -q
|
||||
zstd -f file -q
|
||||
cmp file.zst file-0.zst || die "Level 0 is not the default level"
|
||||
|
||||
# Test level clamping
|
||||
zstd -99 file -o file-99.zst -q
|
||||
cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19"
|
||||
zstd --fast=200000 file -c | zstd -t
|
||||
/*stdin*\ : 65537 bytes
|
||||
|
||||
zstd -5000000000 -f file && die "Level too large, must fail" ||:
|
||||
error: numeric value overflows 32-bit unsigned int
|
||||
zstd --fast=5000000000 -f file && die "Level too large, must fail" ||:
|
||||
error: numeric value overflows 32-bit unsigned int
|
||||
|
||||
# Test setting a level through the environment variable
|
||||
ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst -q
|
||||
ZSTD_CLEVEL=1 zstd file -o file-1-env.zst -q
|
||||
ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst -q
|
||||
ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst -q
|
||||
|
||||
cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
|
||||
cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
|
||||
cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level"
|
||||
cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
|
||||
|
||||
# Test invalid environment clevel is the default level
|
||||
zstd -f file -q
|
||||
ZSTD_CLEVEL=- zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
|
||||
Ignore environment variable setting ZSTD_CLEVEL=-: not a valid integer value
|
||||
ZSTD_CLEVEL=+ zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
|
||||
Ignore environment variable setting ZSTD_CLEVEL=+: not a valid integer value
|
||||
ZSTD_CLEVEL=a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
|
||||
Ignore environment variable setting ZSTD_CLEVEL=a: not a valid integer value
|
||||
ZSTD_CLEVEL=-a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
|
||||
Ignore environment variable setting ZSTD_CLEVEL=-a: not a valid integer value
|
||||
ZSTD_CLEVEL=+a zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
|
||||
Ignore environment variable setting ZSTD_CLEVEL=+a: not a valid integer value
|
||||
ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
|
||||
Ignore environment variable setting ZSTD_CLEVEL=3a7: not a valid integer value
|
||||
ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst -q ; cmp file.zst file-env.zst
|
||||
Ignore environment variable setting ZSTD_CLEVEL=5000000000: numeric value too large
|
||||
|
||||
# Test environment clevel is overridden by command line
|
||||
ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst -q
|
||||
ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst -q
|
||||
|
||||
cmp file-1.zst file-1-env.zst || die "Environment variable not overridden"
|
||||
cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
# Test --long
|
||||
zstd -f file --long ; zstd -t file.zst
|
||||
zstd -f file --long=20; zstd -t file.zst
|
||||
15
build_amd64/_deps/zstd-src/tests/cli-tests/compression/multi-threaded.sh
Executable file
15
build_amd64/_deps/zstd-src/tests/cli-tests/compression/multi-threaded.sh
Executable file
@@ -0,0 +1,15 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
# Test multi-threaded flags
|
||||
zstd --single-thread file -f -q ; zstd -t file.zst
|
||||
zstd -T2 -f file -q ; zstd -t file.zst
|
||||
zstd --rsyncable -f file -q ; zstd -t file.zst
|
||||
zstd -T0 -f file -q ; zstd -t file.zst
|
||||
zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst
|
||||
zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst
|
||||
|
||||
# multi-thread decompression warning test
|
||||
zstd -T0 -f file -q ; zstd -t file.zst; zstd -T0 -d file.zst -o file3
|
||||
zstd -T0 -f file -q ; zstd -t file.zst; zstd -T2 -d file.zst -o file4
|
||||
@@ -0,0 +1,11 @@
|
||||
file.zst : 65537 bytes
|
||||
file.zst : 65537 bytes
|
||||
file.zst : 65537 bytes
|
||||
file.zst : 65537 bytes
|
||||
file.zst : 65537 bytes
|
||||
file.zst : 65537 bytes
|
||||
file.zst : 65537 bytes
|
||||
file.zst : 65537 bytes
|
||||
file.zst : 65537 bytes
|
||||
Warning : decompression does not support multi-threading
|
||||
file.zst : 65537 bytes
|
||||
21
build_amd64/_deps/zstd-src/tests/cli-tests/compression/multiple-files.sh
Executable file
21
build_amd64/_deps/zstd-src/tests/cli-tests/compression/multiple-files.sh
Executable file
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
# setup
|
||||
echo "file1" > file1
|
||||
echo "file2" > file2
|
||||
|
||||
echo "Test zstd ./file1 - file2"
|
||||
rm -f ./file*.zst
|
||||
echo "stdin" | zstd ./file1 - ./file2 | zstd -d
|
||||
cat file1.zst | zstd -d
|
||||
cat file2.zst | zstd -d
|
||||
|
||||
echo "Test zstd -d ./file1.zst - file2.zst"
|
||||
rm ./file1 ./file2
|
||||
echo "stdin" | zstd - | zstd -d ./file1.zst - file2.zst
|
||||
cat file1
|
||||
cat file2
|
||||
|
||||
echo "zstd -d ./file1.zst - file2.zst -c"
|
||||
echo "stdin" | zstd | zstd -d ./file1.zst - file2.zst -c
|
||||
@@ -0,0 +1,12 @@
|
||||
Test zstd ./file1 - file2
|
||||
stdin
|
||||
file1
|
||||
file2
|
||||
Test zstd -d ./file1.zst - file2.zst
|
||||
stdin
|
||||
file1
|
||||
file2
|
||||
zstd -d ./file1.zst - file2.zst -c
|
||||
file1
|
||||
stdin
|
||||
file2
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
# Test --[no-]row-match-finder
|
||||
zstd file -7f --row-match-finder
|
||||
zstd file -7f --no-row-match-finder
|
||||
7
build_amd64/_deps/zstd-src/tests/cli-tests/compression/setup
Executable file
7
build_amd64/_deps/zstd-src/tests/cli-tests/compression/setup
Executable file
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
datagen > file
|
||||
datagen > file0
|
||||
datagen > file1
|
||||
7
build_amd64/_deps/zstd-src/tests/cli-tests/compression/stream-size.sh
Executable file
7
build_amd64/_deps/zstd-src/tests/cli-tests/compression/stream-size.sh
Executable file
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
# Test stream size & hint
|
||||
datagen -g7654 | zstd --stream-size=7654 | zstd -t
|
||||
datagen -g7654 | zstd --size-hint=7000 | zstd -t
|
||||
11
build_amd64/_deps/zstd-src/tests/cli-tests/compression/verbose-wlog.sh
Executable file
11
build_amd64/_deps/zstd-src/tests/cli-tests/compression/verbose-wlog.sh
Executable file
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
. "$COMMON/platform.sh"
|
||||
|
||||
zstd < file -vv -19 -o file.19.zst
|
||||
zstd -vv -l file.19.zst
|
||||
|
||||
zstd < file -vv -19 --long -o file.19.long.zst
|
||||
zstd -vv -l file.19.long.zst
|
||||
@@ -0,0 +1,5 @@
|
||||
...
|
||||
*wlog=23*
|
||||
...
|
||||
*wlog=27*
|
||||
...
|
||||
@@ -0,0 +1,5 @@
|
||||
...
|
||||
*Window Size: 8388608 B*
|
||||
...
|
||||
*Window Size: 134217728 B*
|
||||
...
|
||||
9
build_amd64/_deps/zstd-src/tests/cli-tests/compression/window-resize.sh
Executable file
9
build_amd64/_deps/zstd-src/tests/cli-tests/compression/window-resize.sh
Executable file
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
datagen -g1G > file
|
||||
zstd --long=30 -1 --single-thread --no-content-size -f file
|
||||
zstd -l -v file.zst
|
||||
|
||||
# We want to ignore stderr (its outputting "*** zstd command line interface
|
||||
# 64-bits v1.5.3, by Yann Collet ***")
|
||||
|
||||
rm file file.zst
|
||||
@@ -0,0 +1,3 @@
|
||||
...
|
||||
Window Size: 1.000 GiB (1073741824 B)
|
||||
...
|
||||
11
build_amd64/_deps/zstd-src/tests/cli-tests/decompression/detectErrors.sh
Executable file
11
build_amd64/_deps/zstd-src/tests/cli-tests/decompression/detectErrors.sh
Executable file
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-decompression-errors/"
|
||||
|
||||
for file in "$GOLDEN_DIR"/*; do
|
||||
zstd -t $file && die "should have detected an error"
|
||||
done
|
||||
exit 0
|
||||
|
||||
7
build_amd64/_deps/zstd-src/tests/cli-tests/decompression/golden.sh
Executable file
7
build_amd64/_deps/zstd-src/tests/cli-tests/decompression/golden.sh
Executable file
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-decompression/"
|
||||
|
||||
zstd -r -t "$GOLDEN_DIR"
|
||||
57
build_amd64/_deps/zstd-src/tests/cli-tests/decompression/pass-through.sh
Executable file
57
build_amd64/_deps/zstd-src/tests/cli-tests/decompression/pass-through.sh
Executable file
@@ -0,0 +1,57 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
. "$COMMON/platform.sh"
|
||||
|
||||
echo "" > 1
|
||||
echo "2" > 2
|
||||
echo "23" > 3
|
||||
echo "234" > 4
|
||||
echo "some data" > file
|
||||
|
||||
println "+ passthrough enabled"
|
||||
|
||||
zstd file
|
||||
|
||||
# Test short files
|
||||
zstd -dc --pass-through 1 2 3 4
|
||||
|
||||
# Test *cat symlinks
|
||||
zstdcat file
|
||||
"$ZSTD_SYMLINK_DIR/zcat" file
|
||||
"$ZSTD_SYMLINK_DIR/gzcat" file
|
||||
|
||||
# Test multiple files with mix of compressed & not
|
||||
zstdcat file file.zst
|
||||
zstdcat file.zst file
|
||||
|
||||
# Test --pass-through
|
||||
zstd -dc --pass-through file
|
||||
zstd -d --pass-through file -o pass-through-file
|
||||
|
||||
# Test legacy implicit passthrough with -fc
|
||||
zstd -dcf file
|
||||
zstd -dcf file file.zst
|
||||
zstd -df < file
|
||||
zstd -dcf < file file.zst -
|
||||
zstd -dcf < file.zst file -
|
||||
|
||||
$DIFF file pass-through-file
|
||||
|
||||
println "+ passthrough disabled"
|
||||
|
||||
# Test *cat
|
||||
zstdcat --no-pass-through file && die "should fail"
|
||||
"$ZSTD_SYMLINK_DIR/zcat" --no-pass-through file && die "should fail"
|
||||
"$ZSTD_SYMLINK_DIR/gzcat" --no-pass-through file && die "should fail"
|
||||
# Test zstd without implicit passthrough
|
||||
zstd -d file -o no-pass-through-file && die "should fail"
|
||||
zstd -d < file && die "should fail"
|
||||
|
||||
# Test legacy implicit passthrough with -fc
|
||||
zstd --no-pass-through -dcf file && die "should fail"
|
||||
zstd --no-pass-through -dcf file file.zst && die "should fail"
|
||||
zstd --no-pass-through -df < file && die "should fail"
|
||||
zstd --no-pass-through -dcf < file file.zst - && die "should fail"
|
||||
zstd --no-pass-through -dcf < file.zst file - && die "should fail" ||:
|
||||
@@ -0,0 +1,11 @@
|
||||
file :230.00% ( 10 B => 23 B, file.zst)
|
||||
zstd: file: unsupported format
|
||||
zstd: file: unsupported format
|
||||
zstd: file: unsupported format
|
||||
zstd: file: unsupported format
|
||||
zstd: /*stdin*\: unsupported format
|
||||
zstd: file: unsupported format
|
||||
zstd: file: unsupported format
|
||||
zstd: /*stdin*\: unsupported format
|
||||
zstd: /*stdin*\: unsupported format
|
||||
zstd: file: unsupported format
|
||||
@@ -0,0 +1,25 @@
|
||||
+ passthrough enabled
|
||||
|
||||
2
|
||||
23
|
||||
234
|
||||
some data
|
||||
some data
|
||||
some data
|
||||
some data
|
||||
some data
|
||||
some data
|
||||
some data
|
||||
some data
|
||||
some data
|
||||
some data
|
||||
some data
|
||||
some data
|
||||
some data
|
||||
some data
|
||||
some data
|
||||
some data
|
||||
+ passthrough disabled
|
||||
some data
|
||||
some data
|
||||
some data
|
||||
9
build_amd64/_deps/zstd-src/tests/cli-tests/dict-builder/empty-input.sh
Executable file
9
build_amd64/_deps/zstd-src/tests/cli-tests/dict-builder/empty-input.sh
Executable file
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
for i in $(seq 50); do
|
||||
datagen -s$i > file$i
|
||||
done
|
||||
touch empty
|
||||
|
||||
set -v
|
||||
zstd -q --train empty file*
|
||||
@@ -0,0 +1 @@
|
||||
zstd -q --train empty file*
|
||||
3
build_amd64/_deps/zstd-src/tests/cli-tests/dict-builder/no-inputs.sh
Executable file
3
build_amd64/_deps/zstd-src/tests/cli-tests/dict-builder/no-inputs.sh
Executable file
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -v
|
||||
zstd --train
|
||||
@@ -0,0 +1 @@
|
||||
14
|
||||
@@ -0,0 +1,5 @@
|
||||
zstd --train
|
||||
! Warning : nb of samples too low for proper processing !
|
||||
! Please provide _one file per sample_.
|
||||
! Alternatively, split files into fixed-size blocks representative of samples, with -B#
|
||||
Error 14 : nb of samples too low
|
||||
@@ -0,0 +1,29 @@
|
||||
#!/bin/sh
|
||||
|
||||
. "$COMMON/platform.sh"
|
||||
|
||||
set -e
|
||||
|
||||
if [ false ]; then
|
||||
for seed in $(seq 100); do
|
||||
datagen -g1000 -s$seed > file$seed
|
||||
done
|
||||
|
||||
zstd --train -r . -o dict0 -qq
|
||||
|
||||
for seed in $(seq 101 200); do
|
||||
datagen -g1000 -s$seed > file$seed
|
||||
done
|
||||
|
||||
zstd --train -r . -o dict1 -qq
|
||||
|
||||
[ "$($MD5SUM < dict0)" != "$($MD5SUM < dict1)" ] || die "dictionaries must not match"
|
||||
|
||||
datagen -g1000 -s0 > file0
|
||||
fi
|
||||
|
||||
set -v
|
||||
zstd files/0 -D dicts/0 -q
|
||||
zstd -t files/0.zst -D dicts/0
|
||||
zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
|
||||
zstd -t files/0.zst && die "Must fail" ||:
|
||||
@@ -0,0 +1,7 @@
|
||||
zstd files/0 -D dicts/0 -q
|
||||
zstd -t files/0.zst -D dicts/0
|
||||
files/0.zst : 1000 bytes
|
||||
zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
|
||||
files/0.zst : Decoding error (36) : Dictionary mismatch
|
||||
zstd -t files/0.zst && die "Must fail" ||:
|
||||
files/0.zst : Decoding error (36) : Dictionary mismatch
|
||||
9
build_amd64/_deps/zstd-src/tests/cli-tests/dictionaries/golden.sh
Executable file
9
build_amd64/_deps/zstd-src/tests/cli-tests/dictionaries/golden.sh
Executable file
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
GOLDEN_COMP_DIR="$ZSTD_REPO_DIR/tests/golden-compression/"
|
||||
GOLDEN_DICT_DIR="$ZSTD_REPO_DIR/tests/golden-dictionaries/"
|
||||
|
||||
zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" "$GOLDEN_COMP_DIR/http" -o http.zst
|
||||
zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" -t http.zst
|
||||
6
build_amd64/_deps/zstd-src/tests/cli-tests/dictionaries/setup
Executable file
6
build_amd64/_deps/zstd-src/tests/cli-tests/dictionaries/setup
Executable file
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
cp -r ../files .
|
||||
cp -r ../dicts .
|
||||
24
build_amd64/_deps/zstd-src/tests/cli-tests/dictionaries/setup_once
Executable file
24
build_amd64/_deps/zstd-src/tests/cli-tests/dictionaries/setup_once
Executable file
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
. "$COMMON/platform.sh"
|
||||
|
||||
|
||||
mkdir files/ dicts/
|
||||
|
||||
for seed in $(seq 50); do
|
||||
datagen -g1000 -s$seed > files/$seed
|
||||
done
|
||||
|
||||
zstd --train -r files -o dicts/0 -qq
|
||||
|
||||
for seed in $(seq 51 100); do
|
||||
datagen -g1000 -s$seed > files/$seed
|
||||
done
|
||||
|
||||
zstd --train -r files -o dicts/1 -qq
|
||||
|
||||
cmp dicts/0 dicts/1 && die "dictionaries must not match!"
|
||||
|
||||
datagen -g1000 > files/0
|
||||
49
build_amd64/_deps/zstd-src/tests/cli-tests/file-handling/directory-mirror.sh
Executable file
49
build_amd64/_deps/zstd-src/tests/cli-tests/file-handling/directory-mirror.sh
Executable file
@@ -0,0 +1,49 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
# setup
|
||||
mkdir -p src/.hidden src/dir
|
||||
mkdir mid dst
|
||||
|
||||
echo "file1" > src/file1
|
||||
echo "file2" > src/.file2
|
||||
echo "file3" > src/.hidden/.file3
|
||||
echo "file4" > src/dir/.file4
|
||||
|
||||
# relative paths
|
||||
zstd -q -r --output-dir-mirror mid/ src/
|
||||
zstd -q -d -r --output-dir-mirror dst/ mid/src/
|
||||
|
||||
diff --brief --recursive --new-file src/ dst/mid/src/
|
||||
|
||||
# reset
|
||||
rm -rf mid dst
|
||||
mkdir mid dst
|
||||
|
||||
# from inside the directory
|
||||
(cd src; zstd -q -r --output-dir-mirror ../mid/ ./)
|
||||
(cd mid; zstd -q -d -r --output-dir-mirror ../dst/ ./)
|
||||
|
||||
diff --brief --recursive --new-file src/ dst/
|
||||
|
||||
# reset
|
||||
rm -rf mid dst
|
||||
mkdir mid dst
|
||||
|
||||
# absolute paths
|
||||
export BASE_PATH="$(pwd)"
|
||||
|
||||
zstd -q -r --output-dir-mirror mid/ "${BASE_PATH}/src/"
|
||||
zstd -q -d -r --output-dir-mirror dst/ "${BASE_PATH}/mid/${BASE_PATH}/src/"
|
||||
|
||||
diff --brief --recursive --new-file src/ "dst/${BASE_PATH}/mid/${BASE_PATH}/src/"
|
||||
|
||||
# reset
|
||||
rm -rf mid dst
|
||||
mkdir mid dst
|
||||
|
||||
# dots
|
||||
zstd -q -r --output-dir-mirror mid/ ./src/./
|
||||
zstd -q -d -r --output-dir-mirror dst/ ./mid/./src/./
|
||||
|
||||
diff --brief --recursive --new-file src/ dst/mid/src/
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
|
||||
# motivated by issue #3523
|
||||
|
||||
datagen > file
|
||||
mkdir out
|
||||
chmod 000 out
|
||||
|
||||
zstd file -q --trace-file-stat -o out/file.zst
|
||||
zstd -tq out/file.zst
|
||||
|
||||
chmod 777 out
|
||||
@@ -0,0 +1,26 @@
|
||||
Trace:FileStat: > UTIL_isLink(file)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isConsole(2)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_getFileSize(file)
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: < 65537
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_isDirectoryStat()
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_isSameFile(file, out/file.zst)
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_stat(-1, out/file.zst)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isRegularFile(out/file.zst)
|
||||
Trace:FileStat: > UTIL_stat(-1, out/file.zst)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
zstd: out/file.zst: Permission denied
|
||||
zstd: can't stat out/file.zst : Permission denied -- ignored
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
datagen > file
|
||||
chmod 642 file
|
||||
|
||||
zstd file -q --trace-file-stat -o file.zst
|
||||
zstd -tq file.zst
|
||||
@@ -0,0 +1,42 @@
|
||||
Trace:FileStat: > UTIL_isLink(file)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isConsole(2)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_getFileSize(file)
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: < 65537
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_isDirectoryStat()
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_isSameFile(file, file.zst)
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_stat(-1, file.zst)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isRegularFile(file.zst)
|
||||
Trace:FileStat: > UTIL_stat(-1, file.zst)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isRegularFile(file.zst)
|
||||
Trace:FileStat: > UTIL_stat(-1, file.zst)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_getFileSize(file)
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: < 65537
|
||||
Trace:FileStat: > UTIL_setFileStat(4, file.zst)
|
||||
Trace:FileStat: > UTIL_stat(4, file.zst)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_chmod(file.zst, 0642)
|
||||
Trace:FileStat: > fchmod
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_utime(file.zst)
|
||||
Trace:FileStat: < 0
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
datagen > file
|
||||
|
||||
zstd file -cq --trace-file-stat > file.zst
|
||||
zstd -tq file.zst
|
||||
@@ -0,0 +1,24 @@
|
||||
Trace:FileStat: > UTIL_isLink(file)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isConsole(1)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isConsole(2)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_getFileSize(file)
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: < 65537
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_isDirectoryStat()
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
|
||||
Trace:FileStat: > UTIL_stat(-1, /*stdout*\)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_getFileSize(file)
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: < 65537
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
datagen > file
|
||||
|
||||
zstd < file -q --trace-file-stat -o file.zst
|
||||
zstd -tq file.zst
|
||||
@@ -0,0 +1,24 @@
|
||||
Trace:FileStat: > UTIL_isConsole(0)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isConsole(2)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
|
||||
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < -1
|
||||
Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file.zst)
|
||||
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isRegularFile(file.zst)
|
||||
Trace:FileStat: > UTIL_stat(-1, file.zst)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isRegularFile(file.zst)
|
||||
Trace:FileStat: > UTIL_stat(-1, file.zst)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
|
||||
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < -1
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
datagen > file
|
||||
|
||||
zstd < file -cq --trace-file-stat > file.zst
|
||||
zstd -tq file.zst
|
||||
@@ -0,0 +1,18 @@
|
||||
Trace:FileStat: > UTIL_isConsole(0)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isConsole(1)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isConsole(2)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
|
||||
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < -1
|
||||
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
|
||||
Trace:FileStat: > UTIL_stat(-1, /*stdout*\)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
|
||||
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < -1
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
datagen | zstd -q > file.zst
|
||||
chmod 642 file.zst
|
||||
|
||||
zstd -dq --trace-file-stat file.zst
|
||||
@@ -0,0 +1,38 @@
|
||||
Trace:FileStat: > UTIL_isLink(file.zst)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isConsole(1)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isConsole(2)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isDirectory(file.zst)
|
||||
Trace:FileStat: > UTIL_stat(-1, file.zst)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_isDirectoryStat()
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_stat(-1, file.zst)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_isSameFile(file.zst, file)
|
||||
Trace:FileStat: > UTIL_stat(-1, file.zst)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isRegularFile(file)
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isRegularFile(file)
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_setFileStat(4, file)
|
||||
Trace:FileStat: > UTIL_stat(4, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_chmod(file, 0642)
|
||||
Trace:FileStat: > fchmod
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_utime(file)
|
||||
Trace:FileStat: < 0
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
datagen | zstd -q > file.zst
|
||||
|
||||
zstd -dcq --trace-file-stat file.zst > file
|
||||
@@ -0,0 +1,18 @@
|
||||
Trace:FileStat: > UTIL_isLink(file.zst)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isConsole(1)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isConsole(2)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isDirectory(file.zst)
|
||||
Trace:FileStat: > UTIL_stat(-1, file.zst)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_isDirectoryStat()
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_stat(-1, file.zst)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
|
||||
Trace:FileStat: > UTIL_stat(-1, /*stdout*\)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
datagen | zstd -q > file.zst
|
||||
|
||||
zstd -dcq --trace-file-stat < file.zst -o file
|
||||
@@ -0,0 +1,20 @@
|
||||
Trace:FileStat: > UTIL_isConsole(0)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isConsole(2)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
|
||||
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file)
|
||||
Trace:FileStat: > UTIL_stat(-1, /*stdin*\)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isRegularFile(file)
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: < 0
|
||||
Trace:FileStat: > UTIL_isRegularFile(file)
|
||||
Trace:FileStat: > UTIL_stat(-1, file)
|
||||
Trace:FileStat: < 1
|
||||
Trace:FileStat: < 1
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
|
||||
datagen | zstd -q > file.zst
|
||||
|
||||
zstd -dcq --trace-file-stat < file.zst > file
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user